From 3f62ecf0af77d9708ff8c72b17c43482b9afaac5 Mon Sep 17 00:00:00 2001 From: HeiGeAi <221471965+HeiGeAi@users.noreply.github.com> Date: Fri, 24 Jul 2026 00:24:26 +0800 Subject: [PATCH 1/2] release: prepare Immortal Memory v1.3.0 --- CHANGELOG.md | 9 + README.md | 38 +++- core/VERSION | 2 +- core/config.py | 7 + core/control_data.py | 38 ++++ core/export_restore.py | 1 + core/external_sources.py | 394 +++++++++++++++++++++++++++++++++ core/immortal.py | 75 +++++++ core/orchestrator.py | 45 ++++ docs/ROADMAP.md | 7 + docs/releases/v1.3.0.md | 32 +++ pyproject.toml | 2 +- tests/test_control_data.py | 2 +- tests/test_external_sources.py | 218 ++++++++++++++++++ tests/test_version.py | 4 +- 15 files changed, 865 insertions(+), 9 deletions(-) create mode 100644 core/external_sources.py create mode 100644 docs/releases/v1.3.0.md create mode 100644 tests/test_external_sources.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 16894f6..9299f2f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,14 @@ # Changelog +## 1.3.0 + +- Add explicit, disabled-by-default imports for Git commit history, GitHub pull requests and issues, Claude Web exports, ChatGPT exports, and Cursor transcripts. +- Keep external collection local, read-only, incremental, secret-redacted, and limited to user-registered paths. +- Surface each external source and Feishu Mail separately in the control center without exposing local paths. +- Include external-source deduplication state in portable recovery exports. +- Run enabled external sources through the normal orchestrator and report partial or failed collection honestly. +- Preserve every v1.1 Living Self, transaction, migration, recovery, and dashboard capability by releasing from the current `main` architecture. + ## 1.1.1 - Mark a loaded daily LaunchAgent as requiring attention when its most recent exit code is nonzero, and expose the code as bounded scheduler evidence instead of showing a false healthy state. diff --git a/README.md b/README.md index 845f353..407f306 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@
-![Version](https://img.shields.io/badge/version-v1.1.1-111827.svg) +![Version](https://img.shields.io/badge/version-v1.3.0-111827.svg) ![Python](https://img.shields.io/badge/python-3.9%2B-3776AB.svg?logo=python&logoColor=white) ![Platform](https://img.shields.io/badge/platform-Codex%20%7C%20Claude%20Code%20%7C%20Local%20Agent-0F766E.svg) ![License](https://img.shields.io/badge/license-MIT-059669.svg) @@ -27,7 +27,7 @@ 要让它说出像你会说的话、按你会做的判断去做事，只有一个办法：用**足够高密度的个人上下文**，把它从共识里硬拽出来。 -Immortal Memory 就是干这件事的本地系统。它不是一句 prompt，也不只是一个 Codex 技能。v1.1 用五个可追溯层完成这件事： +Immortal Memory 就是干这件事的本地系统。它不是一句 prompt，也不只是一个 Codex 技能。v1.3 在 v1.1 Living Self 架构上，用五个可追溯层完成这件事： 1. **Claim**：把原始痕迹变成带出处、作用域、归因和隐私标签的可纠正主张。 2. **Living Self**：只用已确认 Claim 生成有版本的「当前自我」，旧版本永不原地覆盖。 @@ -110,6 +110,36 @@ immortal-memory agent-entry immortal-memory agent-context "help me review this product idea" --print ``` +### 自动接入你的高质量语料 + +v1.3 支持五类显式登记的外部来源。全部默认关闭，只有你登记的文件或目录才会被读取： + +```bash +# 本地 Git 提交历史，只读取提交元数据和提交说明，不读取代码正文 +immortal-memory source register git "/absolute/path/to/projects" + +# GitHub PR 和 Issue，只读调用本机已登录的 gh CLI +immortal-memory source register github "/absolute/path/to/projects" + +# 官方导出的对话文件或 Cursor transcript 目录 +immortal-memory source register claude-web "/absolute/path/to/conversations.json" +immortal-memory source register chatgpt "/absolute/path/to/conversations.json" +immortal-memory source register cursor "/absolute/path/to/cursor/agent-transcripts" + +# 检查登记范围并立即执行一次增量采集 +immortal-memory source list --json +immortal-memory source collect --json +``` + +每条记录在写入前都会做凭证形态脱敏，并用本地 SQLite 状态去重。每日编排会自动调用已启用来源。控制台只展示来源健康、更新时间、新增数和错误数，不展示本机私有路径。飞书邮件继续保持显式授权，不会因为安装产品而自动采集。 + +### 支持环境与 Agent + +- macOS：完整支持，包括 LaunchAgent 每日自动化、Codex 和 Claude Code 适配器。 +- Linux：核心 CLI、HTTP Agent Bridge、MCP、采集和测试受支持；需要自行配置 systemd 或 cron 调度。 +- Windows：当前不支持生产运行，因为索引锁依赖 `fcntl.flock`。建议使用 WSL2，并自行配置调度。 +- Agent：Codex、Claude Code、终端 Agent，以及任何能调用 CLI、HTTP 或 MCP 的本地 Agent。适配器只负责发现入口，真实记忆仍由同一套核心权限和上下文合同控制。 + 打开本地控制台： ```bash @@ -212,7 +242,7 @@ immortal-memory agent-context "release acceptance" --print ```bash CLEAN_HOME="$(mktemp -d /tmp/immortal-clean-home.XXXXXX)" python3 -m venv "$CLEAN_HOME/venv" -WHEEL="$(find "$(pwd)/dist" -maxdepth 1 -name 'immortal_memory-1.1.1-*.whl' | head -n 1)" +WHEEL="$(find "$(pwd)/dist" -maxdepth 1 -name 'immortal_memory-1.3.0-*.whl' | head -n 1)" HOME="$CLEAN_HOME" "$CLEAN_HOME/venv/bin/python" -m pip install "$WHEEL" HOME="$CLEAN_HOME" "$CLEAN_HOME/venv/bin/immortal-memory" init --owner-display-name "Clean Install" --alias "clean" HOME="$CLEAN_HOME" "$CLEAN_HOME/venv/bin/immortal-memory" train --smoke @@ -314,7 +344,7 @@ There is a sharper problem underneath. A large model is wired to produce **confi To make it say what you would say and decide the way you would decide, there is only one move: push **high-density personal context** at it until it gets dragged out of the consensus prior. -Immortal Memory is the local system that does exactly that. It is not a prompt, and not only a Codex skill. v1.1 implements five traceable layers: +Immortal Memory is the local system that does exactly that. It is not a prompt, and not only a Codex skill. v1.3 builds on the v1.1 Living Self architecture and implements five traceable layers: 1. **Claim** turns source traces into correctable assertions with evidence, scope, attribution, and privacy labels. 2. **Living Self** builds a versioned current model from confirmed Claims without overwriting history. diff --git a/core/VERSION b/core/VERSION index 524cb55..f0bb29e 100644 --- a/core/VERSION +++ b/core/VERSION @@ -1 +1 @@ -1.1.1 +1.3.0 diff --git a/core/config.py b/core/config.py index fb076de..7f4758d 100644 --- a/core/config.py +++ b/core/config.py @@ -100,6 +100,13 @@ }, }, "extra_sources": [], + "external_sources": { + "git": {"enabled": False, "paths": [], "max_commits": 200}, + "github": {"enabled": False, "paths": [], "max_items": 50}, + "claude-web": {"enabled": False, "paths": []}, + "chatgpt": {"enabled": False, "paths": []}, + "cursor": {"enabled": False, "paths": []}, + }, } diff --git a/core/control_data.py b/core/control_data.py index 48fe5c5..0719f82 100644 --- a/core/control_data.py +++ b/core/control_data.py @@ -394,6 +394,13 @@ def _source_status(value: Any, *, has_success: bool = False) -> str: def sources(self) -> dict[str, Any]: state = self._read_json(self.immortal_dir / "orchestrator_state.json") web = self._read_json(self.immortal_dir / "web" / "state.json") + config = self._read_json(self.immortal_dir / "config.json") + feishu = config.get("feishu") if isinstance(config.get("feishu"), dict) else {} + daily_sources = {item.strip() for item in str(feishu.get("daily_sources") or "").split(",") if item.strip()} + external_config = config.get("external_sources") if isinstance(config.get("external_sources"), dict) else {} + external_state = self._read_json(self.immortal_dir / "external_sources" / "state.json") + external_results = external_state.get("sources") if isinstance(external_state.get("sources"), dict) else {} + external_last = str(external_state.get("generated_at") or "") source_specs = [ { "id": "local", @@ -416,6 +423,37 @@ def sources(self) -> dict[str, Any]: "errors": sum("feishu" in str(item) for item in (state.get("errors") or [])), "evidence": "feishu/state.json", }, + { + "id": "feishu-mail", + "label": "飞书邮件（显式授权）", + "last": str(state.get("last_feishu_collect") or "") if "mail" in daily_sources else "", + "status": self._source_status(state.get("last_feishu_status"), has_success=bool(state.get("last_feishu_collect"))) + if "mail" in daily_sources else "skipped", + "increment": 0, + "errors": 0, + "evidence": "config.json + feishu/state.json", + }, + *[ + { + "id": source_id, + "label": label, + "last": external_last if enabled else "", + "status": self._source_status(result.get("status"), has_success=bool(external_last)) if enabled else "skipped", + "increment": int(result.get("records_written") or 0), + "errors": int(result.get("error_count") or 0), + "evidence": "config.json + external_sources/state.json", + } + for kind, source_id, label in ( + ("git", "git-history", "Git 本地历史"), + ("github", "github-history", "GitHub PR / Issue"), + ("claude-web", "claude-web", "Claude Web 导出"), + ("chatgpt", "chatgpt", "ChatGPT 导出"), + ("cursor", "cursor", "Cursor 导出"), + ) + for configured in [external_config.get(kind) if isinstance(external_config.get(kind), dict) else {}] + for enabled in [bool(configured.get("enabled") and configured.get("paths"))] + for result in [external_results.get(kind) if isinstance(external_results.get(kind), dict) else {}] + ], { "id": "web", "label": "网页访问", diff --git a/core/export_restore.py b/core/export_restore.py index 07a9c64..128c092 100644 --- a/core/export_restore.py +++ b/core/export_restore.py @@ -54,6 +54,7 @@ "people", "quality", "relationships", + "external_sources", # timeline.html / dashboard.html / brief 已于 2026-06-14 停用并删除； # 留在白名单里会让每个新导出都带 missing 警告，strict 校验对完好数据永久 FAIL "digests", diff --git a/core/external_sources.py b/core/external_sources.py new file mode 100644 index 0000000..49b5d9c --- /dev/null +++ b/core/external_sources.py @@ -0,0 +1,394 @@ +#!/usr/bin/env python3 +"""Read-only imports for external sources registered by an explicit local path.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import re +import sqlite3 +import subprocess +from datetime import datetime, timezone +from pathlib import Path +from typing import Any, Iterable + +from index_writer import append_jsonl_records +from maintenance_gate import writer_access +from redact_common import redact_tree + + +SUPPORTED_KINDS = ("git", "github", "claude-web", "chatgpt", "cursor") + + +def iso_utc(value: Any = None) -> str: + if isinstance(value, (int, float)): + return datetime.fromtimestamp(value, tz=timezone.utc).isoformat().replace("+00:00", "Z") + text = str(value or "").strip() + return text or datetime.now(timezone.utc).isoformat().replace("+00:00", "Z") + + +def register_source(settings: dict[str, Any], kind: str, path: str | Path) -> dict[str, Any]: + if kind not in SUPPORTED_KINDS: + raise ValueError(f"unsupported source kind: {kind}") + resolved = Path(path).expanduser().resolve() + if not resolved.exists(): + raise ValueError(f"source path does not exist: {resolved}") + source = settings.setdefault("external_sources", {}).setdefault(kind, {}) + paths = [str(Path(item).expanduser().resolve()) for item in source.get("paths") or []] + if str(resolved) not in paths: + paths.append(str(resolved)) + source.update({"enabled": True, "paths": paths}) + return settings + + +def unregister_source(settings: dict[str, Any], kind: str, path: str | Path) -> dict[str, Any]: + resolved = str(Path(path).expanduser().resolve()) + external = settings.get("external_sources") if isinstance(settings.get("external_sources"), dict) else {} + source = external.get(kind) if isinstance(external.get(kind), dict) else {} + source["paths"] = [ + item for item in source.get("paths") or [] + if str(Path(item).expanduser().resolve()) != resolved + ] + source["enabled"] = bool(source["paths"]) + return settings + + +def _inside(path: Path, root: Path) -> bool: + try: + path.resolve().relative_to(root.resolve()) + return True + except (OSError, ValueError): + return False + + +def discover_git_repositories(root: str | Path, *, max_depth: int = 4) -> list[Path]: + resolved = Path(root).expanduser().resolve() + if (resolved / ".git").exists(): + return [resolved] + repositories: set[Path] = set() + for current, dirs, _files in os.walk(resolved, followlinks=False): + current_path = Path(current) + try: + depth = len(current_path.relative_to(resolved).parts) + except ValueError: + dirs[:] = [] + continue + if depth >= max_depth: + dirs[:] = [] + dirs[:] = [name for name in dirs if name not in {".git", "node_modules", ".venv", "venv"}] + if (current_path / ".git").exists() and _inside(current_path, resolved): + repositories.add(current_path.resolve()) + dirs[:] = [] + return sorted(repositories) + + +def _connection(vault: Path) -> sqlite3.Connection: + state_dir = vault / "external_sources" + state_dir.mkdir(parents=True, exist_ok=True) + connection = sqlite3.connect(state_dir / "state.sqlite3") + connection.execute( + "create table if not exists seen (source text not null, item_key text not null, " + "first_seen_at text not null, primary key(source,item_key))" + ) + return connection + + +def _record(*, source: str, record_type: str, timestamp: Any, content: str, + item_id: str, role: str = "", project: str = "", session_id: str = "", + metadata: dict[str, Any] | None = None) -> dict[str, Any]: + record_id = hashlib.sha256(f"{source}|{item_id}".encode()).hexdigest()[:24] + return { + "id": f"{source}-{record_id}", "source": source, "project": project, + "session_id": session_id, "timestamp": iso_utc(timestamp), "type": record_type, + "role": role, "content": content, "metadata": metadata or {}, "_dedup_key": item_id, + } + + +def _append(vault: Path, records: Iterable[dict[str, Any]]) -> int: + clean = [redact_tree(item) for item in records] + if not clean: + return 0 + by_day: dict[str, list[dict[str, Any]]] = {} + for item in clean: + try: + day = datetime.fromisoformat(str(item["timestamp"]).replace("Z", "+00:00")).astimezone().strftime("%Y-%m-%d") + except ValueError: + day = datetime.now().astimezone().strftime("%Y-%m-%d") + by_day.setdefault(day, []).append(item) + with writer_access(vault): + append_jsonl_records(vault / "index.jsonl", clean, maintenance_held=True) + daily = vault / "daily" + daily.mkdir(parents=True, exist_ok=True) + for day, items in by_day.items(): + with (daily / f"{day}.jsonl").open("a", encoding="utf-8") as handle: + for item in items: + handle.write(json.dumps({k: v for k, v in item.items() if not k.startswith("_")}, ensure_ascii=False) + "\n") + return len(clean) + + +def _collect_records(vault: Path, records: Iterable[dict[str, Any]]) -> int: + connection = _connection(vault) + pending: list[dict[str, Any]] = [] + try: + for item in records: + cursor = connection.execute( + "insert or ignore into seen(source,item_key,first_seen_at) values(?,?,?)", + (item["source"], item["_dedup_key"], iso_utc()), + ) + if cursor.rowcount: + pending.append(item) + written = _append(vault, pending) + connection.commit() + return written + finally: + connection.close() + + +def _git_records(repo: Path, max_commits: int) -> list[dict[str, Any]]: + result = subprocess.run( + ["git", "log", f"--max-count={max(1, int(max_commits))}", "--date=iso-strict", + "--format=%H%x1f%aI%x1f%an%x1f%ae%x1f%B%x1e"], + cwd=repo, text=True, capture_output=True, timeout=60, + ) + if result.returncode: + return [] + records = [] + for block in result.stdout.split("\x1e"): + fields = block.strip().split("\x1f", 4) + if len(fields) != 5: + continue + commit, timestamp, author_name, author_email, message = fields + records.append(_record( + source="git-history", record_type="git-commit", timestamp=timestamp, + content=message.strip(), item_id=f"{repo.resolve()}|{commit}", role="author", + project=repo.name, session_id=commit, + metadata={"repository": repo.name, "commit": commit, + "author_name": author_name, "author_email": author_email}, + )) + return records + + +def collect_git_history(vault: str | Path, roots: Iterable[str | Path], *, max_commits: int = 200) -> dict[str, Any]: + repositories: set[Path] = set() + for root in roots: + repositories.update(discover_git_repositories(root)) + records = [item for repo in sorted(repositories) for item in _git_records(repo, max_commits)] + return {"repositories": len(repositories), "records_written": _collect_records(Path(vault).expanduser(), records), "status": "success"} + + +def github_slug_from_remote(remote: str) -> str: + match = re.match(r"(?:https://github\.com/|git@github\.com:)([^/\s]+/[^/\s]+?)(?:\.git)?$", str(remote or "").strip()) + return match.group(1) if match else "" + + +def _github_remote(repo: Path) -> str: + result = subprocess.run(["git", "config", "--get", "remote.origin.url"], cwd=repo, + text=True, capture_output=True, timeout=15) + return github_slug_from_remote(result.stdout) if result.returncode == 0 else "" + + +def _gh_list(repository: str, kind: str, limit: int) -> tuple[list[dict[str, Any]], bool]: + command = "pr" if kind == "pull-request" else "issue" + fields = "number,title,body,state,createdAt,updatedAt,url" + try: + result = subprocess.run( + ["gh", command, "list", "--repo", repository, "--state", "all", "--limit", str(limit), "--json", fields], + text=True, capture_output=True, timeout=90, + ) + except (OSError, subprocess.TimeoutExpired): + return [], False + if result.returncode: + if kind == "issue" and "disabled issues" in result.stderr.lower(): + return [], True + return [], False + try: + data = json.loads(result.stdout) + except json.JSONDecodeError: + return [], False + return (data if isinstance(data, list) else []), True + + +def github_items_to_records(repository: str, kind: str, items: Iterable[dict[str, Any]]) -> list[dict[str, Any]]: + records = [] + for item in items: + if not isinstance(item, dict) or not item.get("number"): + continue + updated = str(item.get("updatedAt") or item.get("createdAt") or "") + title, body = str(item.get("title") or "").strip(), str(item.get("body") or "").strip() + records.append(_record( + source="github-history", record_type=kind, timestamp=updated, + content=title + (f"\n\n{body}" if body else ""), + item_id=f"{repository}|{kind}|{item['number']}|{updated}", role="author", + project=repository, session_id=f"{kind}-{item['number']}", + metadata={"repository": repository, "number": int(item["number"]), + "state": str(item.get("state") or ""), "url": str(item.get("url") or "")}, + )) + return records + + +def collect_github_history(vault: str | Path, roots: Iterable[str | Path], *, max_items: int = 50) -> dict[str, Any]: + repositories: set[Path] = set() + for root in roots: + repositories.update(discover_git_repositories(root)) + records, errors, count = [], 0, 0 + for repo in sorted(repositories): + slug = _github_remote(repo) + if not slug: + continue + count += 1 + for kind in ("pull-request", "issue"): + items, ok = _gh_list(slug, kind, max(1, int(max_items))) + if ok: + records.extend(github_items_to_records(slug, kind, items)) + else: + errors += 1 + return {"repositories": count, "records_written": _collect_records(Path(vault).expanduser(), records), + "status": "partial" if errors else "success", "error_count": errors} + + +def parse_chatgpt_export(path: str | Path) -> list[dict[str, Any]]: + data = json.loads(Path(path).expanduser().read_text(encoding="utf-8")) + records = [] + for conversation in data if isinstance(data, list) else []: + if not isinstance(conversation, dict): + continue + cid, title = str(conversation.get("id") or conversation.get("conversation_id") or ""), str(conversation.get("title") or "") + mapping = conversation.get("mapping") if isinstance(conversation.get("mapping"), dict) else {} + for node in mapping.values(): + message = node.get("message") if isinstance(node, dict) and isinstance(node.get("message"), dict) else None + if not message: + continue + content = message.get("content") if isinstance(message.get("content"), dict) else {} + text = "\n".join(part for part in content.get("parts") or [] if isinstance(part, str)).strip() + if not text: + continue + mid = str(message.get("id") or hashlib.sha256(text.encode()).hexdigest()[:16]) + author = message.get("author") if isinstance(message.get("author"), dict) else {} + records.append(_record(source="chatgpt-conversation", record_type="conversation-message", + timestamp=message.get("create_time") or conversation.get("create_time"), content=text, + item_id=f"{cid}|{mid}", role=str(author.get("role") or ""), project=title, + session_id=cid, metadata={"conversation_title": title})) + return sorted(records, key=lambda item: item["timestamp"]) + + +def parse_claude_web_export(path: str | Path) -> list[dict[str, Any]]: + data = json.loads(Path(path).expanduser().read_text(encoding="utf-8")) + records = [] + for conversation in data if isinstance(data, list) else []: + if not isinstance(conversation, dict): + continue + cid, title = str(conversation.get("uuid") or ""), str(conversation.get("name") or "") + for message in conversation.get("chat_messages") or []: + if not isinstance(message, dict): + continue + text = str(message.get("text") or "").strip() + if not text: + text = "\n".join(str(block.get("text") or "") for block in message.get("content") or [] + if isinstance(block, dict) and block.get("type") == "text").strip() + if not text: + continue + sender = str(message.get("sender") or "") + role = "user" if sender in {"human", "user"} else "assistant" if sender == "assistant" else sender + mid = str(message.get("uuid") or hashlib.sha256(text.encode()).hexdigest()[:16]) + records.append(_record(source="claude-web-conversation", record_type="conversation-message", + timestamp=message.get("created_at") or conversation.get("created_at"), content=text, + item_id=f"{cid}|{mid}", role=role, project=title, session_id=cid, + metadata={"conversation_title": title})) + return sorted(records, key=lambda item: item["timestamp"]) + + +def parse_cursor_export(path: str | Path) -> list[dict[str, Any]]: + export = Path(path).expanduser() + records = [] + with export.open(encoding="utf-8", errors="ignore") as handle: + for line_no, line in enumerate(handle, 1): + try: + item = json.loads(line) + except json.JSONDecodeError: + continue + if not isinstance(item, dict): + continue + content = item.get("content") + message = item.get("message") if isinstance(item.get("message"), dict) else {} + if not content: + content = "\n".join(str(block.get("text") or "") for block in message.get("content") or [] + if isinstance(block, dict) and block.get("type") == "text") + text = str(content or "").strip() + if not text: + continue + parts = export.parts + project = str(item.get("workspace") or item.get("project") or "") + session = str(item.get("session_id") or "") + if "agent-transcripts" in parts: + marker = parts.index("agent-transcripts") + project = project or (parts[marker - 1] if marker else "") + session = session or export.parent.name + records.append(_record(source="cursor-conversation", record_type="conversation-message", + timestamp=item.get("timestamp"), content=text, + item_id=str(item.get("id") or f"{export.resolve()}:{line_no}"), + role=str(item.get("role") or ""), project=project, session_id=session, + metadata={"import_file": export.name})) + return records + + +def _registered_files(source: dict[str, Any], kind: str) -> list[Path]: + files: set[Path] = set() + for configured in source.get("paths") or []: + path = Path(configured).expanduser().resolve() + if path.is_file(): + files.add(path) + elif path.is_dir(): + pattern = "conversations.json" if kind in {"chatgpt", "claude-web"} else "*.jsonl" + files.update(candidate.resolve() for candidate in path.rglob(pattern) + if candidate.is_file() and _inside(candidate, path)) + return sorted(files) + + +def collect_registered_sources(settings: dict[str, Any], vault: str | Path) -> dict[str, Any]: + vault_path = Path(vault).expanduser() + external = settings.get("external_sources") if isinstance(settings.get("external_sources"), dict) else {} + result: dict[str, Any] = {} + for kind, collector, limit_key, default in ( + ("git", collect_git_history, "max_commits", 200), + ("github", collect_github_history, "max_items", 50), + ): + source = external.get(kind) if isinstance(external.get(kind), dict) else {} + result[kind] = collector(vault_path, source.get("paths") or [], **{limit_key: int(source.get(limit_key) or default)}) \ + if source.get("enabled") and source.get("paths") else {"repositories": 0, "records_written": 0, "status": "disabled"} + for kind, parser in (("claude-web", parse_claude_web_export), ("chatgpt", parse_chatgpt_export), ("cursor", parse_cursor_export)): + source = external.get(kind) if isinstance(external.get(kind), dict) else {} + if not source.get("enabled") or not source.get("paths"): + result[kind] = {"files": 0, "records_written": 0, "status": "disabled"} + continue + files, records, errors = _registered_files(source, kind), [], 0 + for path in files: + try: + records.extend(parser(path)) + except (OSError, ValueError, json.JSONDecodeError): + errors += 1 + result[kind] = {"files": len(files), "records_written": _collect_records(vault_path, records), + "status": "partial" if errors else "success", "error_count": errors} + state_dir = vault_path / "external_sources" + state_dir.mkdir(parents=True, exist_ok=True) + temporary = state_dir / "state.json.tmp" + temporary.write_text(json.dumps({"generated_at": iso_utc(), "sources": result}, ensure_ascii=False, + indent=2, sort_keys=True) + "\n", encoding="utf-8") + temporary.replace(state_dir / "state.json") + return result + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description="Collect explicitly registered external sources") + parser.add_argument("--vault", default=str(Path.home() / ".immortal")) + parser.add_argument("--git-root", action="append", default=[]) + parser.add_argument("--max-commits", type=int, default=200) + args = parser.parse_args(argv) + print(json.dumps(collect_git_history(args.vault, args.git_root, max_commits=args.max_commits), + ensure_ascii=False, sort_keys=True)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/core/immortal.py b/core/immortal.py index e7cd176..b64ea90 100755 --- a/core/immortal.py +++ b/core/immortal.py @@ -45,6 +45,7 @@ migration_backup_gate, restore_check, ) +from external_sources import collect_registered_sources, register_source, unregister_source from index_writer import append_jsonl_records from judgment_store import InvalidJudgmentOperation, JudgmentStore from maintenance_gate import writer_access @@ -224,6 +225,62 @@ def command_run(_args=None) -> int: return run_script("orchestrator.py") +def _external_source_payload(config: dict, kind: str) -> dict: + external = config.get("external_sources") if isinstance(config.get("external_sources"), dict) else {} + source = external.get(kind) if isinstance(external.get(kind), dict) else {} + return { + "kind": kind, + "enabled": bool(source.get("enabled")), + "paths": [str(item) for item in source.get("paths") or []], + } + + +def command_external_source_register(args: argparse.Namespace) -> int: + config = load_config() + try: + register_source(config, args.kind, args.path) + except ValueError as exc: + print(f"Source registration failed: {exc}", file=sys.stderr) + return 2 + save_config(config) + payload = _external_source_payload(config, args.kind) + print(json.dumps(payload, ensure_ascii=False, sort_keys=True) if args.json else f"Registered {args.kind}: {args.path}") + return 0 + + +def command_external_source_unregister(args: argparse.Namespace) -> int: + config = load_config() + unregister_source(config, args.kind, args.path) + save_config(config) + payload = _external_source_payload(config, args.kind) + print(json.dumps(payload, ensure_ascii=False, sort_keys=True) if args.json else f"Unregistered {args.kind}: {args.path}") + return 0 + + +def command_external_source_list(args: argparse.Namespace) -> int: + config = load_config() + payload = {kind: _external_source_payload(config, kind) for kind in ( + "git", "github", "claude-web", "chatgpt", "cursor" + )} + if args.json: + print(json.dumps(payload, ensure_ascii=False, sort_keys=True)) + else: + for kind, source in payload.items(): + print(f"{kind}: {'enabled' if source['enabled'] else 'disabled'} ({len(source['paths'])} paths)") + return 0 + + +def command_external_source_collect(args: argparse.Namespace) -> int: + config = load_config() + payload = collect_registered_sources(config, configured_vault_dir(config)) + if args.json: + print(json.dumps(payload, ensure_ascii=False, sort_keys=True)) + else: + for kind, result in payload.items(): + print(f"{kind}: {int(result.get('records_written') or 0)} new records") + return 0 + + def command_cards(args) -> int: try: store = JudgmentStore(configured_vault_dir()) @@ -1976,6 +2033,24 @@ def build_parser() -> argparse.ArgumentParser: sub = parser.add_subparsers(dest="command") sub.add_parser("status", help="Show current memory library state").set_defaults(func=command_status) + source = sub.add_parser("source", help="Register and collect explicitly scoped external sources") + source_sub = source.add_subparsers(dest="source_command", required=True) + source_list = source_sub.add_parser("list", help="List registered source paths without reading them") + source_list.add_argument("--json", action="store_true") + source_list.set_defaults(func=command_external_source_list) + source_register = source_sub.add_parser("register", help="Register one existing path for a source kind") + source_register.add_argument("kind", choices=["git", "github", "claude-web", "chatgpt", "cursor"]) + source_register.add_argument("path") + source_register.add_argument("--json", action="store_true") + source_register.set_defaults(func=command_external_source_register) + source_unregister = source_sub.add_parser("unregister", help="Remove one registered source path") + source_unregister.add_argument("kind", choices=["git", "github", "claude-web", "chatgpt", "cursor"]) + source_unregister.add_argument("path") + source_unregister.add_argument("--json", action="store_true") + source_unregister.set_defaults(func=command_external_source_unregister) + source_collect = source_sub.add_parser("collect", help="Collect all enabled registered sources") + source_collect.add_argument("--json", action="store_true") + source_collect.set_defaults(func=command_external_source_collect) init = sub.add_parser("init", help="Initialize this installation with the current user's identity and vault config") init.add_argument("--owner-name", default=None, help="Legal or internal name for the owner") init.add_argument("--owner-display-name", default=None, help="Display name used in generated roles") diff --git a/core/orchestrator.py b/core/orchestrator.py index 61c7492..8a11d29 100644 --- a/core/orchestrator.py +++ b/core/orchestrator.py @@ -41,6 +41,7 @@ REQUIRED_FAILURES = { "collect failed", + "external source collect failed", "search index sync failed", "claims migration failed", "context compile failed", @@ -326,6 +327,40 @@ def parse_collect_output(output: str) -> dict: return {"total_new": total_new, "by_source": by_source} +def external_source_collect(): + log("=== 阶段 1E: 受控外部来源同步 ===") + ok, out = run_script("immortal.py", "source", "collect", "--json", timeout=900) + if not ok: + log(f"受控外部来源同步失败: {out.strip()[:300]}") + return False, {"records_written": 0, "sources": {}} + try: + sources = json.loads(out) + except json.JSONDecodeError: + log("受控外部来源同步返回无效 JSON") + return False, {"records_written": 0, "sources": {}} + if not isinstance(sources, dict): + return False, {"records_written": 0, "sources": {}} + total = sum( + int(value.get("records_written") or 0) + for value in sources.values() + if isinstance(value, dict) + ) + partial_sources = sorted( + name for name, value in sources.items() + if isinstance(value, dict) + and str(value.get("status") or "").lower() not in {"success", "disabled", "skipped", "not_due"} + ) + if partial_sources: + log(f"受控外部来源存在异常: {', '.join(partial_sources)}") + return False, { + "records_written": total, + "sources": sources, + "partial_sources": partial_sources, + } + log(f"受控外部来源同步成功: 新增 {total} 条") + return True, {"records_written": total, "sources": sources, "partial_sources": []} + + def days_since(iso_str: str) -> float: """距离指定 ISO 时间戳过了多少天。""" if not iso_str: @@ -1083,6 +1118,13 @@ def run_main(): errors.append("collect failed") telemetry_stage("external", "外部来源同步", errors) + external_ok, external_info = external_source_collect() + external_new = int(external_info.get("records_written") or 0) + if external_ok: + state["last_external_source_collect"] = now_iso + else: + errors.append("external source collect failed") + web_new = 0 web_due_hours = hours_since(state.get("last_web_collect")) if web_due_hours >= WEB_CAPTURE_INTERVAL_HOURS: @@ -1258,6 +1300,7 @@ def run_main(): state["last_run_new_records"] = collect_info.get("total_new", 0) state["last_run_feishu_new_records"] = feishu_new state["last_run_web_new_records"] = web_new + state["last_run_external_new_records"] = external_new state["errors"] = errors[-10:] # 保留最近 10 个错误 save_state(state) @@ -1336,6 +1379,7 @@ def run_main(): log(f"========= 编排器完成 =========") log(f" 本次新增: {collect_info.get('total_new', 0)} 条") + log(f" 受控外部来源新增: {external_new} 条") log(f" 网页新增: {web_new} 条") log(f" 飞书新增: {feishu_new} 条") log(f" 总记录数: {state['total_records']:,}") @@ -1350,6 +1394,7 @@ def run_main(): "errors": errors, "results": { "new_records": collect_info.get("total_new", 0), + "external_new_records": external_new, "web_new_records": web_new, "feishu_new_records": feishu_new, "total_records": state["total_records"], diff --git a/docs/ROADMAP.md b/docs/ROADMAP.md index e8ee017..9243925 100644 --- a/docs/ROADMAP.md +++ b/docs/ROADMAP.md @@ -24,6 +24,13 @@ - Storage budgeting, retention previews, and safer evidence deletion workflows. - Stable local API and MCP transport over the same authorization and context contracts. +## v1.3 Controlled source automation + +- Explicit, disabled-by-default Git, GitHub, Claude Web, ChatGPT, and Cursor sources. +- Incremental local deduplication, secret redaction, bounded collection, and source-level health evidence. +- Feishu Mail remains a separate opt-in source instead of being folded into a generic success state. +- External source state is part of portable recovery and the normal orchestration loop. + ## Later, only after local trust is proven - Additional connector SDKs and audited adapters. diff --git a/docs/releases/v1.3.0.md b/docs/releases/v1.3.0.md new file mode 100644 index 0000000..eee9db6 --- /dev/null +++ b/docs/releases/v1.3.0.md @@ -0,0 +1,32 @@ +# Immortal Memory v1.3.0 + +v1.3.0 turns external knowledge intake into a controlled product capability while preserving the full Living Self architecture already present on `main`. + +## What changed + +- Git commit history can be collected from explicitly registered roots. It reads commit metadata and messages, not repository file bodies. +- GitHub pull requests and issues can be collected read-only through an already authenticated `gh` CLI. +- Claude Web and ChatGPT official exports, plus explicit Cursor JSONL transcripts, can be imported locally. +- Every connector is disabled by default, limited to registered paths, incrementally deduplicated, and passed through credential-shape redaction before persistence. +- The control center reports each connector separately without returning local filesystem paths. +- Feishu Mail is shown as an explicit opt-in source. +- External-source state is included in portable recovery exports and enabled sources run in the normal orchestrator. + +## Compatibility + +- Python 3.9 through 3.12 are covered by CI. +- macOS is the complete production target, including LaunchAgent automation. +- Linux supports the core CLI, HTTP bridge, MCP, collection, and tests. Scheduling must be configured separately. +- Native Windows is not a production target because the index lock uses `fcntl.flock`. WSL2 is the practical Windows path. +- Codex, Claude Code, terminal agents, and other local agents can consume the same bounded context through CLI, HTTP, or MCP. + +## Safety boundaries + +- Registration and collection are local operations. GitHub collection does not fetch, push, comment, merge, or modify repository state. +- Chat exports are never discovered globally. A user must register an existing file or directory. +- A successful local recovery export is not automatically a share-safe artifact. Offsite delivery still requires encryption and a verified restore drill. +- A missing ChatGPT export is reported as disabled or skipped, not as success. + +## Verification contract + +The release is accepted only after the private-data scan, Python compilation, smoke test, full regression suite, P0 scenarios, wheel installation in an isolated home, remote CI, and a clean-clone check all succeed. diff --git a/pyproject.toml b/pyproject.toml index 6e34fc8..90fb465 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "immortal-memory" -version = "1.1.1" +version = "1.3.0" description = "Local-first personal memory layer for AI agents" requires-python = ">=3.9" license = "MIT" diff --git a/tests/test_control_data.py b/tests/test_control_data.py index eb58166..2780654 100644 --- a/tests/test_control_data.py +++ b/tests/test_control_data.py @@ -331,7 +331,7 @@ def test_memory_and_source_routes_are_live(tmp_path): server.server_close() assert source_status == 200 - assert len(sources["items"]) == 5 + assert len(sources["items"]) == 11 assert list_status == 200 assert len(page["items"]) == 2 assert detail_status == 200 diff --git a/tests/test_external_sources.py b/tests/test_external_sources.py new file mode 100644 index 0000000..a3023bf --- /dev/null +++ b/tests/test_external_sources.py @@ -0,0 +1,218 @@ +from __future__ import annotations + +import json +import subprocess +from pathlib import Path + +import pytest + +import config +import export_restore +import external_sources +import immortal +import orchestrator +from control_data import ControlData + + +def _git(repo: Path, *args: str) -> str: + result = subprocess.run( + ["git", *args], cwd=repo, check=True, text=True, capture_output=True + ) + return result.stdout.strip() + + +def _seed_repo(root: Path) -> Path: + repo = root / "project-a" + repo.mkdir(parents=True) + _git(repo, "init") + _git(repo, "config", "user.name", "Tester") + _git(repo, "config", "user.email", "tester@example.invalid") + (repo / "README.md").write_text("private file body\n", encoding="utf-8") + _git(repo, "add", "README.md") + _git(repo, "commit", "-m", "record the first decision") + return repo + + +def test_external_sources_are_disabled_by_default(): + assert config.DEFAULT_CONFIG["external_sources"] == { + "git": {"enabled": False, "paths": [], "max_commits": 200}, + "github": {"enabled": False, "paths": [], "max_items": 50}, + "claude-web": {"enabled": False, "paths": []}, + "chatgpt": {"enabled": False, "paths": []}, + "cursor": {"enabled": False, "paths": []}, + } + + +def test_portable_backup_preserves_external_source_state(): + assert "external_sources" in export_restore.REQUIRED_PATHS + + +def test_register_source_requires_explicit_existing_path(tmp_path): + settings: dict = {} + external_sources.register_source(settings, "git", tmp_path) + assert settings["external_sources"]["git"]["paths"] == [str(tmp_path.resolve())] + with pytest.raises(ValueError, match="does not exist"): + external_sources.register_source(settings, "git", tmp_path / "missing") + + +def test_git_collection_is_incremental_and_does_not_read_file_body(tmp_path): + repo = _seed_repo(tmp_path / "repos") + vault = tmp_path / "vault" + + first = external_sources.collect_git_history(vault, [repo.parent], max_commits=20) + second = external_sources.collect_git_history(vault, [repo.parent], max_commits=20) + + assert first["records_written"] == 1 + assert second["records_written"] == 0 + row = json.loads((vault / "index.jsonl").read_text(encoding="utf-8").splitlines()[0]) + assert row["source"] == "git-history" + assert row["content"] == "record the first decision" + assert "private file body" not in json.dumps(row) + + +@pytest.mark.parametrize( + ("remote", "expected"), + [ + ("https://github.com/HeiGeAi/immortal-memory.git", "HeiGeAi/immortal-memory"), + ("git@github.com:HeiGeAi/immortal-memory.git", "HeiGeAi/immortal-memory"), + ("https://example.com/owner/repo.git", ""), + ], +) +def test_github_remote_parser_accepts_only_github(remote, expected): + assert external_sources.github_slug_from_remote(remote) == expected + + +def test_disabled_github_issues_are_a_supported_empty_state(monkeypatch): + monkeypatch.setattr( + external_sources.subprocess, + "run", + lambda *_args, **_kwargs: subprocess.CompletedProcess( + args=[], returncode=1, stdout="", stderr="the repository has disabled issues" + ), + ) + assert external_sources._gh_list("owner/repo", "issue", 50) == ([], True) + + +def test_missing_github_cli_is_reported_as_connector_failure(monkeypatch): + def missing(*_args, **_kwargs): + raise FileNotFoundError("gh") + + monkeypatch.setattr(external_sources.subprocess, "run", missing) + + assert external_sources._gh_list("owner/repo", "pull-request", 50) == ([], False) + + +def test_chatgpt_claude_and_cursor_exports_are_parsed(tmp_path): + chatgpt = tmp_path / "chatgpt.json" + chatgpt.write_text(json.dumps([{ + "id": "c1", "title": "Decision", "mapping": {"n1": {"message": { + "id": "m1", "author": {"role": "user"}, + "content": {"parts": ["Prefer the reversible option"]}, + }}} + }]), encoding="utf-8") + claude = tmp_path / "claude.json" + claude.write_text(json.dumps([{ + "uuid": "c2", "name": "Review", "chat_messages": [{ + "uuid": "m2", "sender": "human", "text": "Keep the release reversible" + }] + }]), encoding="utf-8") + cursor = tmp_path / "cursor.jsonl" + cursor.write_text(json.dumps({ + "id": "m3", "role": "assistant", "content": "Use the current architecture" + }) + "\n", encoding="utf-8") + + assert external_sources.parse_chatgpt_export(chatgpt)[0]["content"] == "Prefer the reversible option" + assert external_sources.parse_claude_web_export(claude)[0]["role"] == "user" + assert external_sources.parse_cursor_export(cursor)[0]["content"] == "Use the current architecture" + + +def test_registered_collection_redacts_secrets_and_is_incremental(tmp_path): + export = tmp_path / "cursor.jsonl" + export.write_text(json.dumps({ + "id": "m1", "role": "user", "content": "token: ghp_" + "a" * 40 + }) + "\n", encoding="utf-8") + settings = {"external_sources": { + "cursor": {"enabled": True, "paths": [str(export)]} + }} + vault = tmp_path / "vault" + + first = external_sources.collect_registered_sources(settings, vault) + second = external_sources.collect_registered_sources(settings, vault) + + assert first["cursor"]["records_written"] == 1 + assert second["cursor"]["records_written"] == 0 + body = (vault / "index.jsonl").read_text(encoding="utf-8") + assert "ghp_" + "a" * 40 not in body + assert "[REDACTED]" in body + + +def test_source_cli_lists_disabled_defaults_without_collecting(monkeypatch, capsys): + monkeypatch.setattr(immortal, "load_config", lambda: config.DEFAULT_CONFIG) + + assert immortal.main(["source", "list", "--json"]) == 0 + + payload = json.loads(capsys.readouterr().out) + assert set(payload) == {"git", "github", "claude-web", "chatgpt", "cursor"} + assert all(not item["enabled"] for item in payload.values()) + + +def test_orchestrator_collects_declared_external_sources(monkeypatch): + calls = [] + payload = { + "git": {"records_written": 2, "status": "success"}, + "cursor": {"records_written": 1, "status": "success"}, + } + monkeypatch.setattr( + orchestrator, + "run_script", + lambda script, *args, **kwargs: calls.append((script, args, kwargs)) or (True, json.dumps(payload)), + ) + + ok, result = orchestrator.external_source_collect() + + assert ok is True + assert result["records_written"] == 3 + assert calls[0][0:2] == ("immortal.py", ("source", "collect", "--json")) + + +def test_orchestrator_does_not_flatten_partial_external_source_to_success(monkeypatch): + payload = { + "git": {"records_written": 2, "status": "success"}, + "github": {"records_written": 1, "status": "partial", "error_count": 1}, + "chatgpt": {"records_written": 0, "status": "disabled"}, + } + monkeypatch.setattr( + orchestrator, + "run_script", + lambda *_args, **_kwargs: (True, json.dumps(payload)), + ) + + ok, result = orchestrator.external_source_collect() + + assert ok is False + assert result["records_written"] == 3 + assert result["partial_sources"] == ["github"] + + +def test_control_center_shows_external_sources_and_mail_without_paths(tmp_path): + (tmp_path / "config.json").write_text(json.dumps({ + "feishu": {"daily_sources": "contacts,messages"}, + "external_sources": { + "git": {"enabled": True, "paths": ["/private/projects"]}, + "chatgpt": {"enabled": False, "paths": []}, + }, + }), encoding="utf-8") + state_dir = tmp_path / "external_sources" + state_dir.mkdir() + (state_dir / "state.json").write_text(json.dumps({ + "generated_at": "2026-07-23T12:00:00Z", + "sources": {"git": {"records_written": 3, "status": "success"}}, + }), encoding="utf-8") + + items = {item["id"]: item for item in ControlData(tmp_path).sources()["items"]} + + assert items["git-history"]["status"] == "success" + assert items["git-history"]["increment"] == 3 + assert items["chatgpt"]["status"] == "skipped" + assert items["feishu-mail"]["status"] == "skipped" + assert "/private" not in json.dumps(items, ensure_ascii=False) diff --git a/tests/test_version.py b/tests/test_version.py index 60025b6..90ea120 100644 --- a/tests/test_version.py +++ b/tests/test_version.py @@ -32,11 +32,11 @@ def test_source_versions_and_cli_are_consistent(self): immortal.main(["--version"]) self.assertEqual(raised.exception.code, 0) - self.assertEqual(version, "1.1.1") + self.assertEqual(version, "1.3.0") self.assertIsNotNone(project_version) self.assertEqual(project_version.group(1), version) self.assertIn(f"version-v{version}-", readme) - self.assertEqual(output.getvalue().strip(), "immortal 1.1.1") + self.assertEqual(output.getvalue().strip(), "immortal 1.3.0") setup_version = subprocess.run( [sys.executable, "setup.py", "--version"], From 896bb21ef2f5e85cd4609cdd3146d9d7416fe16f Mon Sep 17 00:00:00 2001 From: HeiGeAi <221471965+HeiGeAi@users.noreply.github.com> Date: Fri, 24 Jul 2026 00:27:20 +0800 Subject: [PATCH 2/2] test: isolate external source orchestration logs --- tests/test_external_sources.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/test_external_sources.py b/tests/test_external_sources.py index a3023bf..0128b79 100644 --- a/tests/test_external_sources.py +++ b/tests/test_external_sources.py @@ -167,6 +167,7 @@ def test_orchestrator_collects_declared_external_sources(monkeypatch): "run_script", lambda script, *args, **kwargs: calls.append((script, args, kwargs)) or (True, json.dumps(payload)), ) + monkeypatch.setattr(orchestrator, "log", lambda _message: None) ok, result = orchestrator.external_source_collect() @@ -186,6 +187,7 @@ def test_orchestrator_does_not_flatten_partial_external_source_to_success(monkey "run_script", lambda *_args, **_kwargs: (True, json.dumps(payload)), ) + monkeypatch.setattr(orchestrator, "log", lambda _message: None) ok, result = orchestrator.external_source_collect()