From ae58638b6cb51d30a88019f3d3d37213954abdff Mon Sep 17 00:00:00 2001 From: johnlanni Date: Sat, 8 Aug 2026 20:31:11 +0800 Subject: [PATCH 01/17] docs(specs): materialize durable specs for runner-scoped shared HOME Materializes confirmed SPEC-439001..SPEC-439006 from Proposal #439 into repository durable specs: shared runner-scoped runtime HOME, per-job scratch, storage lifecycle integration, and legacy home migration. Signed-off-by: johnlanni --- .../acpx-comment-triggered-workflow/spec.md | 4 +- .../specs/runner-coordinator-agents/spec.md | 9 +- .../specs/runner-shared-runtime-home/spec.md | 103 ++++++++++++++++++ 3 files changed, 112 insertions(+), 4 deletions(-) create mode 100644 issue-spec/specs/runner-shared-runtime-home/spec.md diff --git a/issue-spec/specs/acpx-comment-triggered-workflow/spec.md b/issue-spec/specs/acpx-comment-triggered-workflow/spec.md index 0b0a2cdf..da230d4a 100644 --- a/issue-spec/specs/acpx-comment-triggered-workflow/spec.md +++ b/issue-spec/specs/acpx-comment-triggered-workflow/spec.md @@ -124,7 +124,7 @@ The runner MUST invoke acpx as an external coordinator backend using argv arrays #### Scenario: default sandbox - **WHEN** a runner job dispatches on Linux without `--unsafe-no-sandbox` -- **THEN** acpx SHALL run through bubblewrap with workspace filesystem isolation, temporary `HOME`, `GH_CONFIG_DIR`, `XDG_CONFIG_HOME`, and `CODEX_HOME` when needed, inherited proxy settings, broad token environment variables scrubbed, required system paths read-only, and the managed workspace mounted for writes. +- **THEN** acpx SHALL run through bubblewrap with workspace filesystem isolation, the persistent runner-scoped isolated runtime `HOME` shared by that runner scope's public sessions (never the operator's real HOME) with managed `GH_CONFIG_DIR`, `XDG_CONFIG_HOME`, and `CODEX_HOME` when needed, unique disposable per-job scratch directories for `TMPDIR`, `GOTMPDIR`, `XDG_DATA_HOME`, and `XDG_STATE_HOME`, inherited proxy settings, broad token environment variables scrubbed, required system paths read-only, and the managed workspace mounted for writes. #### Scenario: unsafe mode @@ -152,7 +152,7 @@ The runner MUST invoke acpx as an external coordinator backend using argv arrays - **THEN** the runner SHALL treat acpx as the top-level headless coordinator transport and persist bounded child provenance reported by the coordinator without needing direct access to the native worker runtime. Source SPEC comments: -- https://github.com/higress-group/issue-spec/issues/343#issuecomment-5099614411 +- https://github.com/higress-group/issue-spec/issues/439#issuecomment-5226066261 ### Requirement: durable job state and recovery diff --git a/issue-spec/specs/runner-coordinator-agents/spec.md b/issue-spec/specs/runner-coordinator-agents/spec.md index 6169295b..a690576e 100644 --- a/issue-spec/specs/runner-coordinator-agents/spec.md +++ b/issue-spec/specs/runner-coordinator-agents/spec.md @@ -38,7 +38,7 @@ Source SPEC comments: ### Requirement: Runner mirrors host qoder configuration into the sandbox -The runner MUST mirror the host ~/.qoder/settings.json and the regular files directly inside ~/.qoder/.auth into the sandbox temporary HOME. The .auth mirror MUST be one level only, MUST preserve restrictive source permission bits (defaulting to 0600), MUST remove stale mirrored files, and MUST NOT follow destination symlinks or mirror cache, log, session-state, symlink, directory, or device entries. +The runner MUST mirror the host ~/.qoder/settings.json and the regular files directly inside ~/.qoder/.auth into the sandbox runtime HOME (the persistent runner-scoped isolated HOME exposed to the sandbox as $HOME, referred to below as $TempHome). The .auth mirror MUST be one level only, MUST preserve restrictive source permission bits (defaulting to 0600), MUST remove stale mirrored files, and MUST NOT follow destination symlinks or mirror cache, log, session-state, symlink, directory, or device entries. Because the runtime HOME is shared by concurrent jobs of one runner scope, each mirrored file refresh MUST be atomic and idempotent so a concurrent job never observes a partially written configuration or credential file. #### Scenario: settings.json is mirrored into sandbox TempHome @@ -65,8 +65,13 @@ The runner MUST mirror the host ~/.qoder/settings.json and the regular files dir - **WHEN** the host qoder config contains symlinks, subdirectories, devices, cache, logs, or session state - **THEN** the mirror MUST skip those entries, MUST NOT recurse, and MUST NOT expose them inside the sandbox +#### Scenario: concurrent dispatches refresh the shared mirror safely + +- **WHEN** two jobs of the same runner scope dispatch concurrently and both refresh the qoder mirror in the shared runtime HOME +- **THEN** each mirrored file MUST be replaced atomically so a running agent observes either the previous or the new complete file, never a partial write + Source SPEC comments: -- https://github.com/higress-group/issue-spec/issues/323#issuecomment-5035087599 +- https://github.com/higress-group/issue-spec/issues/439#issuecomment-5226066355 ### Requirement: Runner derives qoder-specific acpx configuration for job dispatch diff --git a/issue-spec/specs/runner-shared-runtime-home/spec.md b/issue-spec/specs/runner-shared-runtime-home/spec.md new file mode 100644 index 00000000..0a65b0aa --- /dev/null +++ b/issue-spec/specs/runner-shared-runtime-home/spec.md @@ -0,0 +1,103 @@ +# runner-shared-runtime-home + +## Purpose + +Define the long-lived behavior contract for this capability. + +Proposal Issues: +- https://github.com/higress-group/issue-spec/issues/439 + +## Requirements + +### Requirement: Runner sessions share one runner-scoped isolated runtime HOME + +The runner MUST give every public session dispatched for one runner scope the same persistent isolated runtime HOME, where the scope identity is the hostname, backend profile realm, canonical repository identity, and runner identity. The runtime HOME MUST never be the operator's real HOME, MUST be created runner-owned with private (0700) permissions strictly confined below the resolved runner workspace root, and MUST fail closed when the resolved path escapes the root, crosses a symlink, or collides with another root entry. Repository content MUST NOT be able to select or redirect the runtime HOME path. Different repositories, runner identities, or backend profile realms MUST NOT share a writable runtime HOME. Session routing MUST continue to use only the exact stored public session ID, ACPX record ID, ACP session ID, and repository binding from durable runner state; the runner MUST NOT infer the most recent or active conversation from runtime HOME contents. + +#### Scenario: concurrent sessions share one runtime HOME + +- **WHEN** two public sessions of the same runner scope dispatch jobs, concurrently or sequentially +- **THEN** both jobs SHALL receive the same runner-scoped runtime HOME while each dispatch keeps its own exact public session ID, ACPX record ID, and managed workspace, and alternating `/resume` operations before and after a runner restart SHALL reconnect each session to its original agent identity + +#### Scenario: different scope cannot reuse the runtime HOME + +- **WHEN** a job is dispatched for a different repository, runner identity, or backend profile realm +- **THEN** the runner SHALL derive a distinct runtime HOME for that scope and SHALL NOT bind another scope's runtime HOME into the sandbox + +#### Scenario: unsafe path resolution fails closed + +- **WHEN** the derived runtime HOME path escapes the workspace root, resolves through a symlink, or overlaps a protected root entry +- **THEN** dispatch SHALL fail with an actionable diagnostic instead of creating or using the unsafe path + +#### Scenario: operator HOME stays untouched + +- **WHEN** any runner job executes in the default sandbox +- **THEN** the operator's real HOME SHALL remain unavailable for general writes and SHALL never be selected as the shared runtime HOME + +Source SPEC comments: +- https://github.com/higress-group/issue-spec/issues/439#issuecomment-5226066432 + +### Requirement: Each dispatched job receives unique disposable scratch directories + +Every dispatched runner job MUST receive its own private scratch directories for `TMPDIR`, `GOTMPDIR`, `XDG_DATA_HOME`, and `XDG_STATE_HOME`, unique per runner job ID and never shared between jobs. Job scratch MUST be removed after the job reaches a terminal state, and scratch left behind by a crashed runner MUST be recovered by a conservative, idempotent reconciliation that never deletes scratch belonging to an active job. Job-private credential material MUST follow the job scratch lifecycle and MUST NOT persist in the shared runtime HOME. + +#### Scenario: scratch is unique and writable + +- **WHEN** two jobs run concurrently in the same runner scope +- **THEN** each job SHALL observe its own sandbox-writable `TMPDIR`, `GOTMPDIR`, `XDG_DATA_HOME`, and `XDG_STATE_HOME` paths that the other job does not receive + +#### Scenario: scratch is removed at terminal completion + +- **WHEN** a job completes, fails, or is cancelled +- **THEN** the runner SHALL remove that job's scratch directories without touching the shared runtime HOME or another job's scratch + +#### Scenario: crash recovery is conservative and idempotent + +- **WHEN** the runner restarts after a crash that left job scratch directories behind +- **THEN** reconciliation SHALL remove only scratch whose owning job is terminal or unknown to durable state, SHALL keep scratch of active jobs, and repeating the pass SHALL be a no-op + +Source SPEC comments: +- https://github.com/higress-group/issue-spec/issues/439#issuecomment-5226066531 + +### Requirement: Shared runtime HOME storage is owner-locked, accounted, and cache-only evictable + +The runner-scoped runtime HOME and per-job scratch MUST be managed physical storage under the same storage owner lock and reconciliation engine that owns session runtimes and PROCESS pools, and reconciliation of these resources MUST stay idempotent across crashes between filesystem and metadata transitions. Storage diagnostics MUST distinguish protected identity/config bytes, rebuildable cache bytes, job scratch bytes, and unknown bytes for the runtime HOME. Storage pressure and cleanup MAY evict rebuildable cache directories and stale job scratch, but MUST NOT remove authentication or configuration state, ACPX index and session mappings, live agent state, active workspaces, or anything required to `/resume` an existing session. Known-expired resources MUST NOT receive a fresh orphan grace period, and an older runner binary reading the newer storage metadata MUST NOT corrupt or silently delete the runtime HOME or its records. + +#### Scenario: storage report classifies runtime HOME bytes + +- **WHEN** an operator runs the storage reconciliation report +- **THEN** the report SHALL show the runtime HOME's protected identity/config bytes, rebuildable cache bytes, job scratch bytes, and unknown bytes as distinct categories without exposing file contents or credentials + +#### Scenario: cache cleanup preserves resume + +- **WHEN** low disk space or an explicit cleanup evicts rebuildable caches from the runtime HOME +- **THEN** eviction SHALL be limited to cache-eligible and stale scratch paths and every existing public session SHALL remain resumable afterwards + +#### Scenario: old binary rollback is safe + +- **WHEN** an older runner binary that predates the shared runtime HOME runs against the upgraded root +- **THEN** the older binary SHALL NOT delete or corrupt the shared runtime HOME, its metadata records, or the storage sidecar's ownership evidence + +Source SPEC comments: +- https://github.com/higress-group/issue-spec/issues/439#issuecomment-5226066600 + +### Requirement: Legacy per-session runtime homes migrate with resume validation + +Upgrading a runner root that contains legacy per-session runtime HOME directories MUST preserve exact `/resume` compatibility for existing sessions. Migration MUST run only while holding exclusive storage owner ownership with the old runner stopped, MUST back up control-plane state and the storage sidecar first, MUST create the runner-scoped runtime HOME with private permissions, MUST import verified ACPX and agent session state while keeping every existing ACPX record ID and agent session identity unchanged, and MUST NOT copy rebuildable cache directories from every legacy session. Legacy runtime homes MUST be preserved until a real resume through the new runtime HOME succeeds, and only then MAY the redundant legacy data be retired through the existing storage reconciler without granting known-expired resources a fresh orphan grace period. When legacy session homes contain conflicting global agent configuration or indexes, migration MUST stop with actionable diagnostics instead of silently choosing one. Migration, retry, and rollback MUST be idempotent, and rolling back to the previous binary MUST NOT corrupt or lose the sidecar or the new runtime HOME metadata. + +#### Scenario: migrated sessions resume with original identity + +- **WHEN** an operator migrates a root with existing per-session runtime homes and then resumes a migrated session +- **THEN** the session SHALL reconnect to its original ACPX record ID and agent session through the shared runtime HOME, and legacy homes SHALL be retired only after that validation succeeds + +#### Scenario: conflicting legacy configuration fails closed + +- **WHEN** two legacy session homes carry conflicting global agent configuration or index files +- **THEN** migration SHALL stop with a diagnostic naming the conflicting sessions and paths and SHALL NOT pick one silently + +#### Scenario: migration retries are idempotent + +- **WHEN** migration is interrupted and re-run, or re-run after it already completed +- **THEN** the repeated run SHALL converge to the same completed state without duplicating imports, corrupting state, or restarting orphan grace for known-expired resources + +Source SPEC comments: +- https://github.com/higress-group/issue-spec/issues/439#issuecomment-5226066701 From 86c247a13b1d74b389d9bb5f0e835050113cc0b4 Mon Sep 17 00:00:00 2001 From: johnlanni Date: Sat, 8 Aug 2026 21:21:12 +0800 Subject: [PATCH 02/17] feat(runner): add runner-scoped runtime home storage foundations Phase 1 of the runner-scoped isolated runtime HOME change: storage package foundations only, no dispatcher/sandbox wiring yet. - runtime_home.go: RuntimeScope (hostname, realm, repo, runner), scope hash with the physical-hash shape, .runner-home/ and .job-scratch/ layouts, and fail-closed 0700 preparation mirroring the dispatcher's private-dir semantics. scope.json pins each home to its scope; mismatches fail closed naming both scopes. - runtime_store.go: the new .storage/runtime.json metadata store with the sidecar Store's exact semantics (flock, atomic writes, missing/ corrupt rebuild with corrupt backup, foreign-root/newer-schema report-only). A separate file keeps v1 binaries safe: they never parse records they cannot classify. - runtime_accounting.go: protected/cache/unknown byte classification for the shared home, the eviction-priority cache dir list, and symlink-skipping measurement for homes and job scratch. - runtime_reconcile.go: owner-locked, idempotent job-scratch reconciliation (active jobs kept, terminal/unknown removed through capability-scoped deletion, foreign names rejected) and cache-only eviction confined to the exact eligible cache dirs. - migrate.go: two-phase legacy per-session home import with content-digest conflict detection (any conflict aborts before any import write), atomic copies with private perms, a monotonic imported/validated/retired ledger, and retirement that routes legacy runtime deletion through the existing engine without fresh orphan grace. - service.go: lazily opened runtime store plus RecordRuntimeHome, RecordJobScratch, CompleteJobScratch, and the migration ledger methods; remove.go gains a reusable capability-scoped tree remover. Refs higress-group/issue-spec#439, design higress-group/issue-spec#440. Signed-off-by: johnlanni --- internal/commentrunner/storage/migrate.go | 684 ++++++++++++++++++ .../commentrunner/storage/migrate_test.go | 576 +++++++++++++++ internal/commentrunner/storage/remove.go | 8 + .../storage/runtime_accounting.go | 150 ++++ .../storage/runtime_accounting_test.go | 118 +++ .../commentrunner/storage/runtime_home.go | 316 ++++++++ .../storage/runtime_home_test.go | 270 +++++++ .../storage/runtime_reconcile.go | 365 ++++++++++ .../storage/runtime_reconcile_test.go | 387 ++++++++++ .../commentrunner/storage/runtime_store.go | 387 ++++++++++ .../storage/runtime_store_test.go | 238 ++++++ internal/commentrunner/storage/service.go | 222 +++++- 12 files changed, 3719 insertions(+), 2 deletions(-) create mode 100644 internal/commentrunner/storage/migrate.go create mode 100644 internal/commentrunner/storage/migrate_test.go create mode 100644 internal/commentrunner/storage/runtime_accounting.go create mode 100644 internal/commentrunner/storage/runtime_accounting_test.go create mode 100644 internal/commentrunner/storage/runtime_home.go create mode 100644 internal/commentrunner/storage/runtime_home_test.go create mode 100644 internal/commentrunner/storage/runtime_reconcile.go create mode 100644 internal/commentrunner/storage/runtime_reconcile_test.go create mode 100644 internal/commentrunner/storage/runtime_store.go create mode 100644 internal/commentrunner/storage/runtime_store_test.go diff --git a/internal/commentrunner/storage/migrate.go b/internal/commentrunner/storage/migrate.go new file mode 100644 index 00000000..17699073 --- /dev/null +++ b/internal/commentrunner/storage/migrate.go @@ -0,0 +1,684 @@ +package storage + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "sort" + "strings" + + "github.com/higress-group/issue-spec/internal/commentrunner/state" +) + +// runtimeBackupName preserves the runtime metadata before the first applied +// migration, alongside the raw state and sidecar first-migration backups. +const runtimeBackupName = "runtime-first.json" + +// migrationDestinationMarker names the existing shared-home file in a +// conflict's Sessions list. It can never collide with a session key, which +// always has the repo#id shape. +const migrationDestinationMarker = "(shared-home)" + +// MigrateHomeOptions controls one legacy-home migration pass. +type MigrateHomeOptions struct { + Scope RuntimeScope + Apply bool +} + +// MigrationConflict identifies one destination relpath produced with +// different content by multiple legacy sessions, or already present at the +// destination with different content. Sessions lists the conflicting session +// keys, plus migrationDestinationMarker when the destination participates. +type MigrationConflict struct { + RelPath string `json:"rel_path"` + Sessions []string `json:"sessions"` +} + +// MigrateHomeReport summarizes one migration or retirement pass. It carries +// paths, counts, and session identities only — never file contents. +type MigrateHomeReport struct { + ScopeHash string `json:"scope_hash"` + LedgerState MigrationState `json:"ledger_state,omitempty"` + ImportedSessions []string `json:"imported_sessions,omitempty"` + SkippedIdentical int `json:"skipped_identical,omitempty"` + CopiedFiles int `json:"copied_files,omitempty"` + Conflicts []MigrationConflict `json:"conflicts,omitempty"` + RetiredLegacy []string `json:"retired_legacy,omitempty"` + Diagnostics []string `json:"diagnostics,omitempty"` +} + +// importSourceSpec maps one legacy runtime subtree into the shared home. +// Only the listed subtrees are ever imported; caches, logs, sockets, and +// per-job mirrored credentials are deliberately excluded. +type importSourceSpec struct { + srcSub string + destSub string + excludeTop map[string]bool + excludeDir map[string]bool +} + +// codexMirroredTopLevel are refreshed from the host on every dispatch, so the +// legacy copies are never imported. +var codexMirroredTopLevel = map[string]bool{ + "auth.json": true, "config.toml": true, "version.json": true, "installation_id": true, +} + +var claudeMirroredTopLevel = map[string]bool{ + "settings.json": true, "settings.local.json": true, ".credentials.json": true, +} + +var qoderMirroredTopLevel = map[string]bool{"settings.json": true} + +var legacyImportSources = []importSourceSpec{ + {srcSub: filepath.Join("home", ".acpx", "sessions"), destSub: filepath.Join("home", ".acpx", "sessions")}, + {srcSub: filepath.Join("home", ".acpx", "queues"), destSub: filepath.Join("home", ".acpx", "queues")}, + {srcSub: "codex", destSub: "codex", excludeTop: codexMirroredTopLevel}, + {srcSub: filepath.Join("home", ".codex"), destSub: filepath.Join("home", ".codex"), excludeTop: codexMirroredTopLevel}, + {srcSub: filepath.Join("home", ".claude"), destSub: filepath.Join("home", ".claude"), excludeTop: claudeMirroredTopLevel}, + {srcSub: filepath.Join("home", ".qoder"), destSub: filepath.Join("home", ".qoder"), excludeTop: qoderMirroredTopLevel, excludeDir: map[string]bool{".auth": true}}, +} + +// legacySessionSource is one selected session's verified legacy runtime root. +type legacySessionSource struct { + key string + root string +} + +// importCandidate is the chosen source file for one destination relpath. +type importCandidate struct { + src string + digest string + exec bool + session string +} + +// migrationPlan is the phase-A result: the exact copies to perform, the +// already-identical destinations, and every detected conflict. +type migrationPlan struct { + copies map[string]importCandidate + skipped int + conflicts []MigrationConflict +} + +// MigrateHome imports legacy per-session agent state into the runner-scoped +// shared runtime home. It runs only under the storage owner, backs up +// control-plane state first, and is two-phase: a full scan across every +// selected session builds the destination plan, and any conflict aborts +// before a single import write. Apply is idempotent; a dry run performs the +// full scan and reports without touching the filesystem or the ledger. +func (s *Service) MigrateHome(ctx context.Context, opts MigrateHomeOptions) (MigrateHomeReport, error) { + if err := opts.Scope.Validate(); err != nil { + return MigrateHomeReport{}, err + } + scopeHash, err := RuntimeScopeHash(opts.Scope) + if err != nil { + return MigrateHomeReport{}, err + } + report := MigrateHomeReport{ScopeHash: scopeHash} + owner, release, err := EnsureOwner(ctx, s.root) + if err != nil { + return report, fmt.Errorf("migration owner: %w", err) + } + defer release() + ctx = WithOwner(ctx, owner) + + if opts.Apply { + if err := s.migrationBackups(); err != nil { + return report, fmt.Errorf("migration backups: %w", err) + } + } + st, err := s.stateLoader(ctx) + if err != nil { + return report, fmt.Errorf("load runner state for migration: %w", err) + } + st.Normalize() + sources := s.selectLegacySessions(st, opts.Scope.Repo, &report) + for _, source := range sources { + report.ImportedSessions = append(report.ImportedSessions, source.key) + } + + sharedRoot := filepath.Join(s.root, RunnerHomesDirName, scopeHash) + if opts.Apply { + if _, err := PrepareRuntimeHome(s.root, opts.Scope); err != nil { + return report, err + } + } else { + // Dry runs validate an existing binding but never create the tree. + if _, statErr := os.Lstat(sharedRoot); statErr == nil { + if err := checkRuntimeScopeBinding(sharedRoot, opts.Scope); err != nil { + return report, err + } + } else if !errors.Is(statErr, os.ErrNotExist) { + return report, fmt.Errorf("inspect runtime home: %w", statErr) + } + } + + plan, err := buildMigrationPlan(sources, sharedRoot) + if err != nil { + return report, err + } + report.Conflicts = plan.conflicts + if len(plan.conflicts) > 0 { + return report, fmt.Errorf("migration has %d conflicting paths; resolve the legacy homes manually before importing", len(plan.conflicts)) + } + report.SkippedIdentical = plan.skipped + if !opts.Apply { + report.CopiedFiles = len(plan.copies) + report.Diagnostics = append(report.Diagnostics, "dry-run: no filesystem or ledger writes performed") + return report, nil + } + if err := applyMigrationPlan(sharedRoot, plan, &report); err != nil { + return report, err + } + // The ledger never regresses: a re-run after validation keeps the advanced + // state while still union-merging the imported session set. + markState := MigrationImported + existing, has, err := s.RuntimeMigrationLedger(ctx, scopeHash) + if err != nil { + return report, err + } + if has && migrationStateRank(existing.State) > migrationStateRank(MigrationImported) { + markState = existing.State + } + if err := s.MarkRuntimeMigration(ctx, MigrationRecord{ + ScopeHash: scopeHash, + State: markState, + ImportedSessions: report.ImportedSessions, + ValidatedSession: existing.ValidatedSession, + }); err != nil { + return report, err + } + ledger, _, err := s.RuntimeMigrationLedger(ctx, scopeHash) + if err != nil { + return report, err + } + report.LedgerState = ledger.State + return report, nil +} + +// RetireLegacyHomes retires the legacy per-session runtime dirs of an +// already validated migration: the matching sidecar session_runtime records +// are marked retired_known (never a fresh orphan grace) and one engine +// reconcile pass removes them through the existing deletion transactions. The +// ledger advances to retired once the pass succeeds; re-runs are idempotent. +func (s *Service) RetireLegacyHomes(ctx context.Context, scope RuntimeScope, apply bool) (MigrateHomeReport, error) { + if err := scope.Validate(); err != nil { + return MigrateHomeReport{}, err + } + scopeHash, err := RuntimeScopeHash(scope) + if err != nil { + return MigrateHomeReport{}, err + } + report := MigrateHomeReport{ScopeHash: scopeHash} + owner, release, err := EnsureOwner(ctx, s.root) + if err != nil { + return report, fmt.Errorf("retire legacy homes owner: %w", err) + } + defer release() + ctx = WithOwner(ctx, owner) + + ledger, ok, err := s.RuntimeMigrationLedger(ctx, scopeHash) + if err != nil { + return report, err + } + if !ok { + return report, fmt.Errorf("migration ledger for scope hash %q not found; run the import first", scopeHash) + } + if ledger.State != MigrationValidated && ledger.State != MigrationRetired { + return report, fmt.Errorf("migration ledger for scope hash %q is %q; a validated resume through the shared home is required before retiring legacy homes", scopeHash, ledger.State) + } + report.LedgerState = ledger.State + report.ImportedSessions = append([]string(nil), ledger.ImportedSessions...) + + st, err := s.stateLoader(ctx) + if err != nil { + return report, fmt.Errorf("load runner state for legacy retirement: %w", err) + } + st.Normalize() + expectedRoots := s.legacyRootsBySession(st, ledger.ImportedSessions, &report) + + // Only sidecar records whose ownership names an imported session and whose + // path matches that session's exact legacy runtime root may retire. For + // sessions already pruned from state the sidecar's proven ownership fields + // are the path evidence. + sidecar := s.store.State() + type retireTarget struct { + id string + sessionKey string + path string + } + var targets []retireTarget + for id, resource := range sidecar.Resources { + if resource.Kind != ResourceKindSessionRuntime || !resource.Owned() { + continue + } + key := state.PublicSessionKey(resource.Repo, resource.PublicSessionID) + if !containsString(ledger.ImportedSessions, key) { + continue + } + if expected, ok := expectedRoots[key]; ok && filepath.Clean(resource.Path) != expected { + report.Diagnostics = append(report.Diagnostics, safeDiagnostic("sidecar record "+id+" path does not match the session runtime root; skipped")) + continue + } + targets = append(targets, retireTarget{id: id, sessionKey: key, path: resource.Path}) + } + sort.Slice(targets, func(i, j int) bool { return targets[i].id < targets[j].id }) + + if !apply { + for _, target := range targets { + report.Diagnostics = append(report.Diagnostics, "would retire legacy runtime for session "+target.sessionKey) + } + report.Diagnostics = append(report.Diagnostics, "dry-run: no filesystem or ledger writes performed") + return report, nil + } + if err := s.store.Update(func(st *StorageState) error { + for _, target := range targets { + current, ok := st.Resources[target.id] + if !ok { + continue + } + switch current.CleanupState { + case CleanupDeleting, CleanupRemoved: + continue + } + current.CleanupState = CleanupRetiredKnown + current.CleanupAttemptID = "" + current.LastCleanupError = "" + st.Resources[target.id] = current + } + return nil + }); err != nil { + return report, fmt.Errorf("mark legacy runtimes retired: %w", err) + } + if _, err := s.ReconcileStorage(ctx, true, false); err != nil { + return report, fmt.Errorf("retire reconcile pass: %w", err) + } + if err := s.MarkRuntimeMigration(ctx, MigrationRecord{ + ScopeHash: scopeHash, + State: MigrationRetired, + ImportedSessions: ledger.ImportedSessions, + ValidatedSession: ledger.ValidatedSession, + }); err != nil { + return report, err + } + report.LedgerState = MigrationRetired + for _, target := range targets { + if _, err := os.Lstat(target.path); errors.Is(err, os.ErrNotExist) { + report.RetiredLegacy = append(report.RetiredLegacy, target.sessionKey) + } else { + report.Diagnostics = append(report.Diagnostics, safeDiagnostic("legacy runtime for session "+target.sessionKey+" preserved by reconciliation")) + } + } + sort.Strings(report.RetiredLegacy) + return report, nil +} + +// migrationBackups performs the first-migration preservation set: the raw +// pre-Normalize runner state (when configured), the sidecar, and the runtime +// metadata. All three are atomic, idempotent, and never overwritten. +func (s *Service) migrationBackups() error { + if _, err := EnsureRawStateBackup(s.root, s.rawStatePath); err != nil { + return err + } + if err := ensureMetadataBackup(s.root, s.store.Path(), sidecarBackupPrefix+".json"); err != nil { + return err + } + runtimeStore, err := s.RuntimeStore() + if err != nil { + return err + } + return ensureMetadataBackup(s.root, runtimeStore.Path(), runtimeBackupName) +} + +// ensureMetadataBackup atomically preserves the first copy of one metadata +// file under `.storage/backups/`, never overwriting an earlier copy. +func ensureMetadataBackup(workspaceRoot, sourcePath, backupName string) error { + data, err := os.ReadFile(sourcePath) + if errors.Is(err, os.ErrNotExist) { + return nil + } + if err != nil { + return fmt.Errorf("read %s for migration backup: %w", backupName, err) + } + if len(bytes.TrimSpace(data)) == 0 { + return nil + } + canonical, err := Canonicalize(workspaceRoot) + if err != nil { + return err + } + backupDir := filepath.Join(canonical, StorageDirName, backupDirName) + if err := os.MkdirAll(backupDir, 0o700); err != nil { + return err + } + target := filepath.Join(backupDir, backupName) + if _, err := os.Lstat(target); err == nil { + return nil + } + return state.WriteAtomic(target, data) +} + +// selectLegacySessions picks the repo's retained public sessions whose legacy +// runtime root exists below this root with a home/ subdir. Sessions without +// workspace metadata or with roots outside this root are skipped with a +// diagnostic: foreign paths are never imported. +func (s *Service) selectLegacySessions(st state.RunnerState, repo string, report *MigrateHomeReport) []legacySessionSource { + keys := make([]string, 0, len(st.PublicSessions)) + for key := range st.PublicSessions { + keys = append(keys, key) + } + sort.Strings(keys) + var sources []legacySessionSource + for _, key := range keys { + session := st.PublicSessions[key] + if session.Repo != repo { + continue + } + wsPath := strings.TrimSpace(session.Workspace.Path) + if wsPath == "" { + report.Diagnostics = append(report.Diagnostics, "session "+key+" lacks workspace metadata; skipped") + continue + } + hash, err := SessionRuntimeHash(session.Repo, session.PublicSessionID, wsPath) + if err != nil { + report.Diagnostics = append(report.Diagnostics, safeDiagnostic("session "+key+" runtime hash failed: "+err.Error())) + continue + } + legacyRoot, err := SessionRuntimeRoot(wsPath, session.Repo, session.PublicSessionID) + if err != nil { + report.Diagnostics = append(report.Diagnostics, safeDiagnostic("session "+key+" runtime root failed: "+err.Error())) + continue + } + if expected := filepath.Join(s.root, SessionsDirName, hash); legacyRoot != expected { + report.Diagnostics = append(report.Diagnostics, safeDiagnostic("session "+key+" legacy runtime root is outside this workspace root; skipped")) + continue + } + if info, statErr := os.Lstat(legacyRoot); statErr != nil || info.Mode()&os.ModeSymlink != 0 || !info.IsDir() { + continue + } + if info, statErr := os.Lstat(filepath.Join(legacyRoot, "home")); statErr != nil || info.Mode()&os.ModeSymlink != 0 || !info.IsDir() { + continue + } + sources = append(sources, legacySessionSource{key: key, root: legacyRoot}) + } + return sources +} + +// legacyRootsBySession recomputes the exact legacy runtime root for imported +// sessions whose workspace metadata survives in state. +func (s *Service) legacyRootsBySession(st state.RunnerState, sessionKeys []string, report *MigrateHomeReport) map[string]string { + roots := map[string]string{} + for _, key := range sessionKeys { + session, ok := st.PublicSessions[key] + if !ok { + continue + } + wsPath := strings.TrimSpace(session.Workspace.Path) + if wsPath == "" { + continue + } + hash, err := SessionRuntimeHash(session.Repo, session.PublicSessionID, wsPath) + if err != nil { + report.Diagnostics = append(report.Diagnostics, safeDiagnostic("session "+key+" runtime hash failed: "+err.Error())) + continue + } + roots[key] = filepath.Join(s.root, SessionsDirName, hash) + } + return roots +} + +// buildMigrationPlan scans every selected session's importable subtrees and +// resolves each destination relpath to one content digest. A relpath produced +// with different digests by two sessions, or already present at the +// destination with a different digest, is a conflict. +func buildMigrationPlan(sources []legacySessionSource, sharedRoot string) (migrationPlan, error) { + plan := migrationPlan{copies: map[string]importCandidate{}} + type occurrence struct { + digest string + session string + } + occurrences := map[string][]occurrence{} + chosen := map[string]importCandidate{} + for _, source := range sources { + for _, spec := range legacyImportSources { + srcBase := filepath.Join(source.root, spec.srcSub) + info, err := os.Lstat(srcBase) + if errors.Is(err, os.ErrNotExist) { + continue + } + if err != nil { + return plan, fmt.Errorf("inspect legacy import source: %w", err) + } + if info.Mode()&os.ModeSymlink != 0 || !info.IsDir() { + continue + } + walkErr := filepath.WalkDir(srcBase, func(path string, entry os.DirEntry, err error) error { + if err != nil { + return err + } + rel, err := filepath.Rel(srcBase, path) + if err != nil { + return err + } + if rel == "." { + return nil + } + slashRel := filepath.ToSlash(rel) + top, _, _ := strings.Cut(slashRel, "/") + depthOne := !strings.Contains(slashRel, "/") + if entry.IsDir() { + if spec.excludeDir[top] || (depthOne && spec.excludeTop[top]) { + return filepath.SkipDir + } + return nil + } + // Symlinks, devices, sockets: never imported. + if !entry.Type().IsRegular() { + return nil + } + if depthOne && spec.excludeTop[top] { + return nil + } + digest, _, err := digestFileBytes(path) + if err != nil { + return err + } + fileInfo, err := entry.Info() + if err != nil { + return err + } + destRel := filepath.Join(spec.destSub, rel) + occurrences[destRel] = append(occurrences[destRel], occurrence{digest: digest, session: source.key}) + if _, ok := chosen[destRel]; !ok { + chosen[destRel] = importCandidate{ + src: path, + digest: digest, + exec: fileInfo.Mode().Perm()&0o111 != 0, + session: source.key, + } + } + return nil + }) + if walkErr != nil { + return plan, fmt.Errorf("scan legacy session %s: %w", source.key, walkErr) + } + } + } + + rels := make([]string, 0, len(occurrences)) + for rel := range occurrences { + rels = append(rels, rel) + } + sort.Strings(rels) + for _, rel := range rels { + occs := occurrences[rel] + digests := map[string]bool{} + sessions := map[string]bool{} + for _, occ := range occs { + digests[occ.digest] = true + sessions[occ.session] = true + } + candidate := chosen[rel] + conflictSessions := func(withDestination bool) []string { + set := map[string]bool{} + for session := range sessions { + set[session] = true + } + if withDestination { + set[migrationDestinationMarker] = true + } + list := make([]string, 0, len(set)) + for session := range set { + list = append(list, session) + } + sort.Strings(list) + return list + } + dest := filepath.Join(sharedRoot, rel) + destInfo, statErr := os.Lstat(dest) + switch { + case errors.Is(statErr, os.ErrNotExist): + // No destination yet. + case statErr != nil: + return plan, fmt.Errorf("inspect migration destination: %w", statErr) + case destInfo.Mode().IsRegular() && destInfo.Mode()&os.ModeSymlink == 0: + destDigest, _, err := digestFileBytes(dest) + if err != nil { + return plan, err + } + if len(digests) == 1 && digests[destDigest] { + // Already imported identically: re-runs converge. + plan.skipped++ + continue + } + plan.conflicts = append(plan.conflicts, MigrationConflict{RelPath: filepath.ToSlash(rel), Sessions: conflictSessions(true)}) + continue + default: + // A symlink, directory, or special file at the destination is + // never overwritten. + plan.conflicts = append(plan.conflicts, MigrationConflict{RelPath: filepath.ToSlash(rel), Sessions: conflictSessions(true)}) + continue + } + if len(digests) > 1 { + plan.conflicts = append(plan.conflicts, MigrationConflict{RelPath: filepath.ToSlash(rel), Sessions: conflictSessions(false)}) + continue + } + plan.copies[rel] = candidate + } + return plan, nil +} + +// applyMigrationPlan performs phase B: atomic temp-and-rename copies into the +// shared home, 0700 directories and 0600 files (0700 when the source is +// executable). Destinations are re-validated against the plan so a concurrent +// change fails closed instead of being overwritten. +func applyMigrationPlan(sharedRoot string, plan migrationPlan, report *MigrateHomeReport) error { + rels := make([]string, 0, len(plan.copies)) + for rel := range plan.copies { + rels = append(rels, rel) + } + sort.Strings(rels) + for _, rel := range rels { + candidate := plan.copies[rel] + dest := filepath.Join(sharedRoot, rel) + info, err := os.Lstat(dest) + switch { + case errors.Is(err, os.ErrNotExist): + case err != nil: + return fmt.Errorf("inspect migration destination: %w", err) + default: + if info.Mode().IsRegular() && info.Mode()&os.ModeSymlink == 0 { + if digest, _, digestErr := digestFileBytes(dest); digestErr == nil && digest == candidate.digest { + report.SkippedIdentical++ + continue + } + } + return fmt.Errorf("migration destination %q changed during import; refusing to overwrite", dest) + } + if err := os.MkdirAll(filepath.Dir(dest), 0o700); err != nil { + return fmt.Errorf("prepare migration destination directory: %w", err) + } + if err := copyFileAtomic(dest, candidate.src, candidate.exec); err != nil { + return fmt.Errorf("copy migration file: %w", err) + } + report.CopiedFiles++ + } + return nil +} + +// copyFileAtomic writes src to dest through a temp file in the destination +// directory plus rename, so a concurrent reader never observes a partial +// import. +func copyFileAtomic(dest, src string, exec bool) error { + in, err := os.Open(src) + if err != nil { + return err + } + defer in.Close() + dir := filepath.Dir(dest) + tmp, err := os.CreateTemp(dir, ".migrate-tmp-*") + if err != nil { + return err + } + tmpName := tmp.Name() + cleanup := true + defer func() { + if cleanup { + os.Remove(tmpName) + } + }() + if _, err := io.Copy(tmp, in); err != nil { + _ = tmp.Close() + return err + } + perm := os.FileMode(0o600) + if exec { + perm = 0o700 + } + if err := tmp.Chmod(perm); err != nil { + _ = tmp.Close() + return err + } + if err := tmp.Sync(); err != nil { + _ = tmp.Close() + return err + } + if err := tmp.Close(); err != nil { + return err + } + if err := os.Rename(tmpName, dest); err != nil { + return err + } + cleanup = false + return nil +} + +// digestFileBytes hashes one regular file without reading it into memory. +func digestFileBytes(path string) (string, int64, error) { + file, err := os.Open(path) + if err != nil { + return "", 0, err + } + defer file.Close() + hash := sha256.New() + size, err := io.Copy(hash, file) + if err != nil { + return "", 0, err + } + return hex.EncodeToString(hash.Sum(nil)), size, nil +} + +func containsString(values []string, needle string) bool { + for _, value := range values { + if value == needle { + return true + } + } + return false +} diff --git a/internal/commentrunner/storage/migrate_test.go b/internal/commentrunner/storage/migrate_test.go new file mode 100644 index 00000000..a7f78357 --- /dev/null +++ b/internal/commentrunner/storage/migrate_test.go @@ -0,0 +1,576 @@ +package storage + +import ( + "context" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/higress-group/issue-spec/internal/commentrunner/state" +) + +type migrateFixture struct { + t *testing.T + root string + svc *Service + st state.RunnerState + scope RuntimeScope + scopeHash string + rawPath string +} + +func writeContent(t *testing.T, path, content string, perm os.FileMode) { + t.Helper() + if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil { + t.Fatalf("mkdir: %v", err) + } + if err := os.WriteFile(path, []byte(content), perm); err != nil { + t.Fatalf("write %q: %v", path, err) + } +} + +func newMigrateFixture(t *testing.T) *migrateFixture { + t.Helper() + root := testRoot(t) + scope := testScope() + hash, err := RuntimeScopeHash(scope) + if err != nil { + t.Fatalf("scope hash: %v", err) + } + f := &migrateFixture{ + t: t, + root: root, + st: state.NewState(), + scope: scope, + scopeHash: hash, + rawPath: filepath.Join(root, "runner-state.json"), + } + writeContent(t, f.rawPath, `{"schema_version":6}`, 0o600) + svc, err := NewService(ServiceConfig{ + WorkspaceRoot: root, + StateLoader: func(context.Context) (state.RunnerState, error) { return f.st, nil }, + RawStatePath: f.rawPath, + OrphanGrace: DefaultOrphanGrace, + }) + if err != nil { + t.Fatalf("NewService: %v", err) + } + t.Cleanup(func() { _ = svc.Close() }) + f.svc = svc + return f +} + +func (f *migrateFixture) addSession(sid string) string { + f.t.Helper() + wsPath := filepath.Join(f.root, "ws-"+sid) + if err := os.MkdirAll(wsPath, 0o700); err != nil { + f.t.Fatalf("mkdir workspace: %v", err) + } + session := terminalSession(f.scope.Repo, sid, "ws-"+sid, wsPath, time.Now().Add(-time.Hour)) + if err := f.st.UpsertPublicSession(session); err != nil { + f.t.Fatalf("upsert session: %v", err) + } + return wsPath +} + +func (f *migrateFixture) legacyRoot(sid string) string { + f.t.Helper() + session := f.st.PublicSessions[state.PublicSessionKey(f.scope.Repo, sid)] + root, err := SessionRuntimeRoot(session.Workspace.Path, f.scope.Repo, sid) + if err != nil { + f.t.Fatalf("legacy root: %v", err) + } + return root +} + +func (f *migrateFixture) sharedHome() string { + return filepath.Join(f.root, RunnerHomesDirName, f.scopeHash) +} + +func (f *migrateFixture) sessionKeys(sids ...string) []string { + keys := make([]string, 0, len(sids)) + for _, sid := range sids { + keys = append(keys, state.PublicSessionKey(f.scope.Repo, sid)) + } + return keys +} + +// seedLegacyHome writes the standard import fixture into one session's legacy +// runtime root: importable agent state, per-job mirrored files that must be +// skipped, and caches that must never be imported. +func seedLegacyHome(t *testing.T, root, indexContent string) { + t.Helper() + writeContent(t, filepath.Join(root, "home", ".acpx", "sessions", "index.json"), indexContent, 0o600) + writeContent(t, filepath.Join(root, "home", ".acpx", "queues", "q1", "item.json"), "queue-item", 0o600) + writeContent(t, filepath.Join(root, "codex", "sessions", "s1.json"), "codex-session", 0o600) + writeContent(t, filepath.Join(root, "codex", "tools", "run.sh"), "#!/bin/sh\n", 0o755) + writeContent(t, filepath.Join(root, "home", ".claude", "projects", "p.json"), "claude-project", 0o600) + writeContent(t, filepath.Join(root, "home", ".qoder", "mcp.json"), "qoder-mcp", 0o600) + // Mirrored per job from the host: never imported. + writeContent(t, filepath.Join(root, "codex", "auth.json"), "codex-auth", 0o600) + writeContent(t, filepath.Join(root, "codex", "config.toml"), "codex-config", 0o600) + writeContent(t, filepath.Join(root, "home", ".codex", "version.json"), "codex-version", 0o600) + writeContent(t, filepath.Join(root, "home", ".codex", "installation_id"), "codex-install", 0o600) + writeContent(t, filepath.Join(root, "home", ".claude", "settings.json"), "claude-settings", 0o600) + writeContent(t, filepath.Join(root, "home", ".claude", "settings.local.json"), "claude-local", 0o600) + writeContent(t, filepath.Join(root, "home", ".claude", ".credentials.json"), "claude-creds", 0o600) + writeContent(t, filepath.Join(root, "home", ".qoder", "settings.json"), "qoder-settings", 0o600) + writeContent(t, filepath.Join(root, "home", ".qoder", ".auth", "token"), "qoder-token", 0o600) + // Rebuildable or runtime-local state: never imported. + writeContent(t, filepath.Join(root, "home", ".cache", "big"), "cache", 0o600) + writeContent(t, filepath.Join(root, "home", "go", "pkg", "mod", "m.zip"), "mod", 0o600) + writeContent(t, filepath.Join(root, "home", ".npm", "registry.tgz"), "npm", 0o600) + writeContent(t, filepath.Join(root, "gh", "hosts.yml"), "gh", 0o600) + writeContent(t, filepath.Join(root, "acpx-runtime", "sock"), "runtime", 0o600) +} + +func readContent(t *testing.T, path string) string { + t.Helper() + data, err := os.ReadFile(path) + if err != nil { + t.Fatalf("read %q: %v", path, err) + } + return string(data) +} + +func TestMigrateHomeHappyImport(t *testing.T) { + f := newMigrateFixture(t) + f.addSession("ps-1") + f.addSession("ps-2") + seedLegacyHome(t, f.legacyRoot("ps-1"), "shared-index") + writeContent(t, filepath.Join(f.legacyRoot("ps-1"), "home", ".acpx", "sessions", "sess-a.json"), "session-a", 0o600) + seedLegacyHome(t, f.legacyRoot("ps-2"), "shared-index") + writeContent(t, filepath.Join(f.legacyRoot("ps-2"), "home", ".acpx", "sessions", "sess-b.json"), "session-b", 0o600) + // ps-2 carries no claude/qoder/codex extras; identical index dedupes. + + report, err := f.svc.MigrateHome(context.Background(), MigrateHomeOptions{Scope: f.scope, Apply: true}) + if err != nil { + t.Fatalf("MigrateHome: %v", err) + } + if report.ScopeHash != f.scopeHash { + t.Fatalf("scope hash = %q, want %q", report.ScopeHash, f.scopeHash) + } + if report.LedgerState != MigrationImported { + t.Fatalf("ledger state = %q, want %q", report.LedgerState, MigrationImported) + } + if len(report.Conflicts) != 0 { + t.Fatalf("unexpected conflicts: %+v", report.Conflicts) + } + for _, key := range f.sessionKeys("ps-1", "ps-2") { + if !contains(report.ImportedSessions, key) { + t.Fatalf("imported sessions %v missing %q", report.ImportedSessions, key) + } + } + // Unique destinations: index deduped, sess-a/sess-b merged, queue item, + // codex session + executable tool, claude project, qoder mcp. + if report.CopiedFiles != 8 { + t.Fatalf("copied = %d, want 8", report.CopiedFiles) + } + home := f.sharedHome() + wantFiles := map[string]string{ + "home/.acpx/sessions/index.json": "shared-index", + "home/.acpx/sessions/sess-a.json": "session-a", + "home/.acpx/sessions/sess-b.json": "session-b", + "home/.acpx/queues/q1/item.json": "queue-item", + "codex/sessions/s1.json": "codex-session", + "home/.claude/projects/p.json": "claude-project", + "home/.qoder/mcp.json": "qoder-mcp", + } + for rel, content := range wantFiles { + got := readContent(t, filepath.Join(home, filepath.FromSlash(rel))) + if got != content { + t.Fatalf("%s = %q, want %q", rel, got, content) + } + } + // Private perms: 0600 files, 0700 for executables. + info, err := os.Lstat(filepath.Join(home, "home", ".acpx", "sessions", "sess-a.json")) + if err != nil || info.Mode().Perm() != 0o600 { + t.Fatalf("imported file perm = %v, err=%v; want 0600", info.Mode(), err) + } + tool := filepath.Join(home, "codex", "tools", "run.sh") + info, err = os.Lstat(tool) + if err != nil || info.Mode().Perm() != 0o700 { + t.Fatalf("imported executable perm = %v, err=%v; want 0700", info.Mode(), err) + } + // Mirrored and cache content must not be imported. + absent := []string{ + "codex/auth.json", "codex/config.toml", + "home/.codex/version.json", "home/.codex/installation_id", + "home/.claude/settings.json", "home/.claude/settings.local.json", "home/.claude/.credentials.json", + "home/.qoder/settings.json", "home/.qoder/.auth", + "home/.cache", "home/go", "home/.npm", + "gh/hosts.yml", "acpx-runtime/sock", + } + for _, rel := range absent { + if _, err := os.Lstat(filepath.Join(home, filepath.FromSlash(rel))); !os.IsNotExist(err) { + t.Fatalf("%s must not be imported, err=%v", rel, err) + } + } + // Scope binding written; ledger recorded; raw state backup taken. + if _, err := os.Lstat(filepath.Join(home, "scope.json")); err != nil { + t.Fatalf("scope.json missing: %v", err) + } + ledger, ok, err := f.svc.RuntimeMigrationLedger(context.Background(), f.scopeHash) + if err != nil || !ok { + t.Fatalf("ledger missing: %v ok=%v", err, ok) + } + if ledger.State != MigrationImported || len(ledger.ImportedSessions) != 2 { + t.Fatalf("ledger = %+v", ledger) + } + if _, err := os.Lstat(filepath.Join(f.root, StorageDirName, backupDirName, rawStateBackupName)); err != nil { + t.Fatalf("raw state backup missing: %v", err) + } + // Legacy homes stay in place until validated retirement. + if _, err := os.Lstat(f.legacyRoot("ps-1")); err != nil { + t.Fatalf("legacy home must be preserved after import: %v", err) + } +} + +func TestMigrateHomeConflictFailsClosed(t *testing.T) { + f := newMigrateFixture(t) + f.addSession("ps-1") + f.addSession("ps-2") + seedLegacyHome(t, f.legacyRoot("ps-1"), "shared-index") + seedLegacyHome(t, f.legacyRoot("ps-2"), "shared-index") + // Conflicting relpath: different bytes in each legacy home. + writeContent(t, filepath.Join(f.legacyRoot("ps-1"), "home", ".claude", "projects", "p.json"), "project-A", 0o600) + writeContent(t, filepath.Join(f.legacyRoot("ps-2"), "home", ".claude", "projects", "p.json"), "project-B", 0o600) + + report, err := f.svc.MigrateHome(context.Background(), MigrateHomeOptions{Scope: f.scope, Apply: true}) + if err == nil { + t.Fatalf("conflicting legacy content must fail migration") + } + if len(report.Conflicts) != 1 { + t.Fatalf("conflicts = %+v, want exactly 1", report.Conflicts) + } + conflict := report.Conflicts[0] + if conflict.RelPath != "home/.claude/projects/p.json" { + t.Fatalf("conflict path = %q", conflict.RelPath) + } + for _, key := range f.sessionKeys("ps-1", "ps-2") { + if !contains(conflict.Sessions, key) { + t.Fatalf("conflict sessions %v missing %q", conflict.Sessions, key) + } + } + // No import writes happened. + home := f.sharedHome() + if _, statErr := os.Lstat(filepath.Join(home, "home", ".claude", "projects", "p.json")); !os.IsNotExist(statErr) { + t.Fatalf("conflicting file must not be imported, err=%v", statErr) + } + if _, statErr := os.Lstat(filepath.Join(home, "codex", "sessions", "s1.json")); !os.IsNotExist(statErr) { + t.Fatalf("no file may be imported when any conflict exists, err=%v", statErr) + } + if _, ok, _ := f.svc.RuntimeMigrationLedger(context.Background(), f.scopeHash); ok { + t.Fatalf("conflicted migration must not write the ledger") + } +} + +func TestMigrateHomeDestinationConflictFailsClosed(t *testing.T) { + f := newMigrateFixture(t) + f.addSession("ps-1") + seedLegacyHome(t, f.legacyRoot("ps-1"), "shared-index") + if _, err := f.svc.MigrateHome(context.Background(), MigrateHomeOptions{Scope: f.scope, Apply: true}); err != nil { + t.Fatalf("first MigrateHome: %v", err) + } + // The destination changed underneath (e.g. a concurrent writer): the next + // import of diverging legacy content must fail, never overwrite. + writeContent(t, filepath.Join(f.sharedHome(), "home", ".qoder", "mcp.json"), "locally-modified", 0o600) + report, err := f.svc.MigrateHome(context.Background(), MigrateHomeOptions{Scope: f.scope, Apply: true}) + if err == nil { + t.Fatalf("diverged destination must fail migration") + } + found := false + for _, conflict := range report.Conflicts { + if conflict.RelPath == "home/.qoder/mcp.json" && contains(conflict.Sessions, migrationDestinationMarker) { + found = true + } + } + if !found { + t.Fatalf("destination conflict must name the shared home marker: %+v", report.Conflicts) + } + if got := readContent(t, filepath.Join(f.sharedHome(), "home", ".qoder", "mcp.json")); got != "locally-modified" { + t.Fatalf("destination must not be overwritten, got %q", got) + } +} + +func TestMigrateHomeApplyTwiceIdempotent(t *testing.T) { + f := newMigrateFixture(t) + f.addSession("ps-1") + seedLegacyHome(t, f.legacyRoot("ps-1"), "shared-index") + first, err := f.svc.MigrateHome(context.Background(), MigrateHomeOptions{Scope: f.scope, Apply: true}) + if err != nil { + t.Fatalf("first MigrateHome: %v", err) + } + if first.CopiedFiles != 6 || first.SkippedIdentical != 0 { + t.Fatalf("first run copied=%d skipped=%d, want 6/0", first.CopiedFiles, first.SkippedIdentical) + } + second, err := f.svc.MigrateHome(context.Background(), MigrateHomeOptions{Scope: f.scope, Apply: true}) + if err != nil { + t.Fatalf("second MigrateHome: %v", err) + } + if second.CopiedFiles != 0 || second.SkippedIdentical != 6 { + t.Fatalf("second run copied=%d skipped=%d, want 0/6", second.CopiedFiles, second.SkippedIdentical) + } + if second.LedgerState != MigrationImported { + t.Fatalf("ledger = %q, want imported", second.LedgerState) + } + // The second run found runtime.json on disk and preserved it once. + if _, err := os.Lstat(filepath.Join(f.root, StorageDirName, backupDirName, runtimeBackupName)); err != nil { + t.Fatalf("runtime metadata backup missing after second run: %v", err) + } +} + +func TestMigrateHomeDryRunDoesNotMutate(t *testing.T) { + f := newMigrateFixture(t) + f.addSession("ps-1") + seedLegacyHome(t, f.legacyRoot("ps-1"), "shared-index") + report, err := f.svc.MigrateHome(context.Background(), MigrateHomeOptions{Scope: f.scope, Apply: false}) + if err != nil { + t.Fatalf("MigrateHome dry-run: %v", err) + } + if report.CopiedFiles != 6 { + t.Fatalf("dry-run would-copy = %d, want 6", report.CopiedFiles) + } + if len(report.ImportedSessions) != 1 { + t.Fatalf("dry-run imported sessions = %v", report.ImportedSessions) + } + if _, err := os.Lstat(filepath.Join(f.root, RunnerHomesDirName)); !os.IsNotExist(err) { + t.Fatalf("dry-run must not create %s, err=%v", RunnerHomesDirName, err) + } + if _, ok, _ := f.svc.RuntimeMigrationLedger(context.Background(), f.scopeHash); ok { + t.Fatalf("dry-run must not write the ledger") + } + if _, err := os.Lstat(filepath.Join(f.root, StorageDirName, backupDirName, rawStateBackupName)); !os.IsNotExist(err) { + t.Fatalf("dry-run must not write backups, err=%v", err) + } +} + +func TestMigrateHomeSkipsSessionsWithoutWorkspace(t *testing.T) { + f := newMigrateFixture(t) + f.addSession("ps-1") + seedLegacyHome(t, f.legacyRoot("ps-1"), "shared-index") + // A session without workspace metadata cannot locate its legacy home. + key := state.PublicSessionKey(f.scope.Repo, "ps-3") + f.st.PublicSessions[key] = state.PublicSession{ + Repo: f.scope.Repo, PublicSessionID: "ps-3", AcpxRecordID: "rec-ps-3", Status: state.StatusCompleted, + } + report, err := f.svc.MigrateHome(context.Background(), MigrateHomeOptions{Scope: f.scope, Apply: true}) + if err != nil { + t.Fatalf("MigrateHome: %v", err) + } + if contains(report.ImportedSessions, key) { + t.Fatalf("session without workspace must not import: %v", report.ImportedSessions) + } + joined := strings.Join(report.Diagnostics, "\n") + if !strings.Contains(joined, key) { + t.Fatalf("diagnostics must explain the skipped session: %v", report.Diagnostics) + } +} + +func TestMigrateHomeLedgerNeverRegresses(t *testing.T) { + f := newMigrateFixture(t) + f.addSession("ps-1") + seedLegacyHome(t, f.legacyRoot("ps-1"), "shared-index") + if _, err := f.svc.MigrateHome(context.Background(), MigrateHomeOptions{Scope: f.scope, Apply: true}); err != nil { + t.Fatalf("MigrateHome: %v", err) + } + if err := f.svc.MarkRuntimeMigration(context.Background(), MigrationRecord{ + ScopeHash: f.scopeHash, State: MigrationValidated, ValidatedSession: state.PublicSessionKey(f.scope.Repo, "ps-1"), + }); err != nil { + t.Fatalf("mark validated: %v", err) + } + // Direct regression is rejected. + if err := f.svc.MarkRuntimeMigration(context.Background(), MigrationRecord{ + ScopeHash: f.scopeHash, State: MigrationImported, + }); err == nil { + t.Fatalf("ledger regression must be rejected") + } + // A migration re-run stays idempotent and keeps the advanced ledger state. + report, err := f.svc.MigrateHome(context.Background(), MigrateHomeOptions{Scope: f.scope, Apply: true}) + if err != nil { + t.Fatalf("re-run MigrateHome: %v", err) + } + if report.LedgerState != MigrationValidated { + t.Fatalf("ledger = %q, want validated after re-run", report.LedgerState) + } + ledger, _, err := f.svc.RuntimeMigrationLedger(context.Background(), f.scopeHash) + if err != nil { + t.Fatalf("ledger: %v", err) + } + if ledger.ValidatedSession != state.PublicSessionKey(f.scope.Repo, "ps-1") { + t.Fatalf("validated session lost: %+v", ledger) + } +} + +func TestRetireLegacyHomesRequiresValidated(t *testing.T) { + f := newMigrateFixture(t) + f.addSession("ps-1") + seedLegacyHome(t, f.legacyRoot("ps-1"), "shared-index") + if _, err := f.svc.MigrateHome(context.Background(), MigrateHomeOptions{Scope: f.scope, Apply: true}); err != nil { + t.Fatalf("MigrateHome: %v", err) + } + if _, err := f.svc.RetireLegacyHomes(context.Background(), f.scope, true); err == nil || !strings.Contains(err.Error(), "validated") { + t.Fatalf("retire with imported ledger must require validated, err=%v", err) + } + other := RuntimeScope{Hostname: "host-1", Repo: "o/other", Runner: "runner-1"} + if _, err := f.svc.RetireLegacyHomes(context.Background(), other, true); err == nil { + t.Fatalf("retire without a ledger must fail") + } +} + +func TestRetireLegacyHomes(t *testing.T) { + f := newMigrateFixture(t) + ws1 := f.addSession("ps-1") + ws2 := f.addSession("ps-2") + seedLegacyHome(t, f.legacyRoot("ps-1"), "shared-index") + seedLegacyHome(t, f.legacyRoot("ps-2"), "shared-index") + if _, err := f.svc.MigrateHome(context.Background(), MigrateHomeOptions{Scope: f.scope, Apply: true}); err != nil { + t.Fatalf("MigrateHome: %v", err) + } + if err := f.svc.MarkRuntimeMigration(context.Background(), MigrationRecord{ + ScopeHash: f.scopeHash, State: MigrationValidated, ValidatedSession: state.PublicSessionKey(f.scope.Repo, "ps-1"), + }); err != nil { + t.Fatalf("mark validated: %v", err) + } + for _, sid := range []string{"ps-1", "ps-2"} { + session := f.st.PublicSessions[state.PublicSessionKey(f.scope.Repo, sid)] + if err := f.svc.RecordSessionResources(context.Background(), f.scope.Repo, sid, session.Workspace.Path); err != nil { + t.Fatalf("RecordSessionResources %s: %v", sid, err) + } + } + // The session clones are gone, so the engine can retire the runtimes. + for _, ws := range []string{ws1, ws2} { + if err := os.RemoveAll(ws); err != nil { + t.Fatalf("remove workspace: %v", err) + } + } + + dry, err := f.svc.RetireLegacyHomes(context.Background(), f.scope, false) + if err != nil { + t.Fatalf("dry-run retire: %v", err) + } + if len(dry.RetiredLegacy) != 0 { + t.Fatalf("dry-run must not retire: %+v", dry) + } + for _, sid := range []string{"ps-1", "ps-2"} { + if _, err := os.Lstat(f.legacyRoot(sid)); err != nil { + t.Fatalf("dry-run must not delete legacy home: %v", err) + } + } + + report, err := f.svc.RetireLegacyHomes(context.Background(), f.scope, true) + if err != nil { + t.Fatalf("RetireLegacyHomes: %v", err) + } + if report.LedgerState != MigrationRetired { + t.Fatalf("ledger = %q, want retired", report.LedgerState) + } + for _, key := range f.sessionKeys("ps-1", "ps-2") { + if !contains(report.RetiredLegacy, key) { + t.Fatalf("retired legacy %v missing %q", report.RetiredLegacy, key) + } + } + for _, sid := range []string{"ps-1", "ps-2"} { + if _, err := os.Lstat(f.legacyRoot(sid)); !os.IsNotExist(err) { + t.Fatalf("legacy home for %s must be deleted, err=%v", sid, err) + } + } + // The shared home survives retirement. + if got := readContent(t, filepath.Join(f.sharedHome(), "home", ".acpx", "sessions", "index.json")); got != "shared-index" { + t.Fatalf("shared home damaged by retirement: %q", got) + } + // Idempotent re-run. + second, err := f.svc.RetireLegacyHomes(context.Background(), f.scope, true) + if err != nil { + t.Fatalf("second RetireLegacyHomes: %v", err) + } + if second.LedgerState != MigrationRetired { + t.Fatalf("second run ledger = %q, want retired", second.LedgerState) + } +} + +func TestV1EngineIgnoresRuntimeHomeResources(t *testing.T) { + f := newMigrateFixture(t) + // The upgraded root carries shared homes, job scratch, and runtime.json. + paths, err := PrepareRuntimeHome(f.root, f.scope) + if err != nil { + t.Fatalf("PrepareRuntimeHome: %v", err) + } + if err := f.svc.RecordRuntimeHome(context.Background(), f.scope, paths); err != nil { + t.Fatalf("RecordRuntimeHome: %v", err) + } + scratch, err := PrepareJobScratch(f.root, scratchJobActive) + if err != nil { + t.Fatalf("PrepareJobScratch: %v", err) + } + if err := f.svc.RecordJobScratch(context.Background(), scratchJobActive, scratch.Root); err != nil { + t.Fatalf("RecordJobScratch: %v", err) + } + writeFile(t, filepath.Join(paths.Home, ".acpx", "sessions", "index.json"), 128) + writeFile(t, filepath.Join(scratch.Tmp, "payload"), 64) + // Plus one legacy orphan the v1 engine must still reclaim. + orphanHash := strings.Repeat("ef", 16) + orphanDir := filepath.Join(f.root, SessionsDirName, orphanHash) + writeFile(t, filepath.Join(orphanDir, "home", "stale"), 32) + + engine, err := NewEngine(EngineConfig{ + WorkspaceRoot: f.root, + StateLoader: func(context.Context) (state.RunnerState, error) { return state.NewState(), nil }, + }) + if err != nil { + t.Fatalf("NewEngine: %v", err) + } + if _, err := engine.Reconcile(context.Background(), ReconcileOptions{Apply: true, OrphanGrace: 0}); err != nil { + t.Fatalf("first Reconcile: %v", err) + } + report, err := engine.Reconcile(context.Background(), ReconcileOptions{Apply: true, OrphanGrace: 0}) + if err != nil { + t.Fatalf("second Reconcile: %v", err) + } + orphanID := ResourceID(ResourceKindSessionRuntime, "", "", orphanHash) + got := reportByID(t, report, orphanID) + if got.Action != ActionDeleted { + t.Fatalf("v1 engine must still delete the legacy orphan, got %+v", got) + } + if _, err := os.Lstat(orphanDir); !os.IsNotExist(err) { + t.Fatalf("legacy orphan must be gone, err=%v", err) + } + // The v1 engine neither inventories nor deletes the new roots. + for _, resource := range report.Resources { + if strings.Contains(resource.ID, RunnerHomesDirName) || strings.Contains(resource.ID, JobScratchDirName) || + strings.Contains(resource.Hash, RunnerHomesDirName) { + t.Fatalf("v1 engine must never inventory runner-home resources: %+v", resource) + } + } + if _, err := os.Lstat(filepath.Join(paths.Home, ".acpx", "sessions", "index.json")); err != nil { + t.Fatalf("shared runtime home damaged by v1 engine: %v", err) + } + if _, err := os.Lstat(filepath.Join(scratch.Tmp, "payload")); err != nil { + t.Fatalf("job scratch damaged by v1 engine: %v", err) + } + // The sidecar stays kind-valid and runtime.json is untouched. + store, err := OpenStore(f.root) + if err != nil { + t.Fatalf("OpenStore: %v", err) + } + defer store.Close() + for id, resource := range store.State().Resources { + if !resource.Kind.Valid() { + t.Fatalf("sidecar resource %q has invalid kind %q", id, resource.Kind) + } + } + runtimeStore, err := OpenRuntimeStore(f.root) + if err != nil { + t.Fatalf("OpenRuntimeStore: %v", err) + } + defer runtimeStore.Close() + runtimeState := runtimeStore.State() + if _, ok := runtimeState.Homes[f.scopeHash]; !ok { + t.Fatalf("runtime home record lost: %+v", runtimeState.Homes) + } + if _, ok := runtimeState.Scratch[scratchJobActive]; !ok { + t.Fatalf("job scratch record lost: %+v", runtimeState.Scratch) + } +} diff --git a/internal/commentrunner/storage/remove.go b/internal/commentrunner/storage/remove.go index 4b023b33..eb100b1c 100644 --- a/internal/commentrunner/storage/remove.go +++ b/internal/commentrunner/storage/remove.go @@ -14,6 +14,14 @@ func RemoveManagedTree(workspaceRoot, target, expectedHash string, beforeRemove if err := ValidateDeletionTarget(workspaceRoot, target, expectedHash); err != nil { return err } + return removeOpenedTree(target, beforeRemove) +} + +// removeOpenedTree removes an already validated target directory through an +// opened root capability. Callers must prove the target's identity and +// confinement first; this helper only guarantees the opened object — not a +// replaced pathname — is what gets removed. +func removeOpenedTree(target string, beforeRemove func()) error { info, err := os.Lstat(target) if os.IsNotExist(err) { return nil diff --git a/internal/commentrunner/storage/runtime_accounting.go b/internal/commentrunner/storage/runtime_accounting.go new file mode 100644 index 00000000..9570dada --- /dev/null +++ b/internal/commentrunner/storage/runtime_accounting.go @@ -0,0 +1,150 @@ +package storage + +import ( + "errors" + "fmt" + "os" + "path/filepath" + "strings" +) + +// RuntimeHomeUsage splits one runtime home's bytes into the diagnostic +// classes: protected identity/config, rebuildable cache, and unknown. +type RuntimeHomeUsage struct { + ProtectedBytes int64 `json:"protected_bytes"` + CacheBytes int64 `json:"cache_bytes"` + UnknownBytes int64 `json:"unknown_bytes"` +} + +// Total sums all classes. +func (u RuntimeHomeUsage) Total() int64 { return u.ProtectedBytes + u.CacheBytes + u.UnknownBytes } + +// RuntimeUsage is the per-root storage accounting view for the shared home +// resources: one home's classification plus total job scratch bytes. +type RuntimeUsage struct { + Home RuntimeHomeUsage `json:"home"` + ScratchBytes int64 `json:"scratch_bytes"` +} + +type runtimeUsageClass int + +const ( + runtimeUsageUnknown runtimeUsageClass = iota + runtimeUsageProtected + runtimeUsageCache +) + +// protectedHomeEntries below home/ hold agent identity and configuration: +// losing them breaks /resume or re-leaks credentials, so they are never +// evicted. Everything else under home/ that is not listed as cache is +// unknown and equally protected from eviction. +var protectedHomeEntries = map[string]bool{ + ".acpx": true, + ".qoder": true, + ".claude": true, + ".codex": true, + ".config": true, + ".ssh": true, + ".gitconfig": true, +} + +// cacheHomeEntries below home/ are rebuildable downloads. +var cacheHomeEntries = map[string]bool{ + ".cache": true, + ".npm": true, + "go": true, +} + +// RuntimeCacheDirs lists the eviction-eligible cache subtrees of one runtime +// home in eviction priority order: the most rebuildable (_npx) first. +func RuntimeCacheDirs(home string) []string { + return []string{ + filepath.Join(home, ".npm", "_npx"), + filepath.Join(home, ".npm"), + filepath.Join(home, ".cache"), + filepath.Join(home, "go", "pkg", "mod"), + } +} + +// classifyRuntimeHomeRel classifies one regular file by its path relative to +// the scope root. The whole go/ tree counts as cache so GOPATH build caches +// and binaries never inflate protected bytes. +func classifyRuntimeHomeRel(rel string) runtimeUsageClass { + rel = filepath.ToSlash(rel) + if rel == runtimeScopeFileName { + return runtimeUsageProtected + } + first, rest, _ := strings.Cut(rel, "/") + switch first { + case "gh", "xdg", "codex", "acpx-runtime": + return runtimeUsageProtected + case "home": + entry, _, _ := strings.Cut(rest, "/") + switch { + case entry == ".claude.json": + return runtimeUsageProtected + case protectedHomeEntries[entry]: + return runtimeUsageProtected + case cacheHomeEntries[entry]: + return runtimeUsageCache + default: + return runtimeUsageUnknown + } + default: + return runtimeUsageUnknown + } +} + +// MeasureRuntimeHome walks one scope root without following symlinks and +// classifies regular-file bytes, the same rules as measureTreeBytes. +func MeasureRuntimeHome(root string) (RuntimeHomeUsage, error) { + var usage RuntimeHomeUsage + walkErr := filepath.WalkDir(root, func(path string, entry os.DirEntry, err error) error { + if err != nil { + return err + } + if entry.Type()&os.ModeSymlink != 0 { + return nil + } + info, infoErr := entry.Info() + if infoErr != nil { + return infoErr + } + if !info.Mode().IsRegular() { + return nil + } + rel, relErr := filepath.Rel(root, path) + if relErr != nil { + return relErr + } + switch classifyRuntimeHomeRel(rel) { + case runtimeUsageProtected: + usage.ProtectedBytes += info.Size() + case runtimeUsageCache: + usage.CacheBytes += info.Size() + default: + usage.UnknownBytes += info.Size() + } + return nil + }) + if walkErr != nil { + return RuntimeHomeUsage{}, fmt.Errorf("measure runtime home %q: %w", root, walkErr) + } + return usage, nil +} + +// MeasureJobScratch sums regular-file bytes below `.job-scratch`, skipping +// symlinks; a missing scratch base measures as zero. +func MeasureJobScratch(workspaceRoot string) (int64, error) { + canonical, err := Canonicalize(workspaceRoot) + if err != nil { + return 0, err + } + base := filepath.Join(canonical, JobScratchDirName) + if _, err := os.Lstat(base); errors.Is(err, os.ErrNotExist) { + return 0, nil + } else if err != nil { + return 0, fmt.Errorf("inspect job scratch base: %w", err) + } + return measureTreeBytes(base), nil +} diff --git a/internal/commentrunner/storage/runtime_accounting_test.go b/internal/commentrunner/storage/runtime_accounting_test.go new file mode 100644 index 00000000..70e148f0 --- /dev/null +++ b/internal/commentrunner/storage/runtime_accounting_test.go @@ -0,0 +1,118 @@ +package storage + +import ( + "os" + "path/filepath" + "reflect" + "testing" +) + +func TestRuntimeCacheDirsOrder(t *testing.T) { + home := filepath.Join(string(filepath.Separator), "scope", "home") + got := RuntimeCacheDirs(home) + want := []string{ + filepath.Join(home, ".npm", "_npx"), + filepath.Join(home, ".npm"), + filepath.Join(home, ".cache"), + filepath.Join(home, "go", "pkg", "mod"), + } + if !reflect.DeepEqual(got, want) { + t.Fatalf("RuntimeCacheDirs = %v, want %v", got, want) + } +} + +func TestMeasureRuntimeHomeClassification(t *testing.T) { + root := t.TempDir() + protected := map[string]int{ + "scope.json": 100, + "gh/hosts.yml": 10, + "xdg/config.toml": 20, + "codex/sessions/s1.json": 30, + "acpx-runtime/state.json": 40, + "home/.acpx/sessions/index.json": 50, + "home/.claude.json": 60, + "home/.gitconfig": 70, + "home/.ssh/id_ed25519": 80, + "home/.codex/sessions/s2.json": 5, + "home/.config/gh/hosts.yml": 15, + "home/.qoder/mcp.json": 25, + "home/.claude/projects/p.json": 8, + } + cache := map[string]int{ + "home/.cache/go-build/abc": 200, + "home/go/pkg/mod/m.zip": 300, + "home/go/bin/tool": 35, + "home/.npm/registry.tgz": 400, + "home/.npm/_npx/pkg/i.js": 50, + } + unknown := map[string]int{ + "home/random.txt": 500, + "home/.local/share/x.db": 600, + "stray.txt": 700, + } + sum := func(files map[string]int) int64 { + var total int64 + for rel, size := range files { + writeFile(t, filepath.Join(root, filepath.FromSlash(rel)), size) + total += int64(size) + } + return total + } + wantProtected := sum(protected) + wantCache := sum(cache) + wantUnknown := sum(unknown) + // Symlinks are never followed or counted, even inside classified trees. + outside := t.TempDir() + writeFile(t, filepath.Join(outside, "payload"), 9999) + if err := os.Symlink(filepath.Join(outside, "payload"), filepath.Join(root, "home", ".acpx", "link")); err != nil { + t.Fatalf("symlink: %v", err) + } + if err := os.Symlink(outside, filepath.Join(root, "home", ".cache", "linkdir")); err != nil { + t.Fatalf("symlink dir: %v", err) + } + + usage, err := MeasureRuntimeHome(root) + if err != nil { + t.Fatalf("MeasureRuntimeHome: %v", err) + } + if usage.ProtectedBytes != wantProtected { + t.Fatalf("protected = %d, want %d", usage.ProtectedBytes, wantProtected) + } + if usage.CacheBytes != wantCache { + t.Fatalf("cache = %d, want %d", usage.CacheBytes, wantCache) + } + if usage.UnknownBytes != wantUnknown { + t.Fatalf("unknown = %d, want %d", usage.UnknownBytes, wantUnknown) + } +} + +func TestMeasureRuntimeHomeMissingRoot(t *testing.T) { + if _, err := MeasureRuntimeHome(filepath.Join(t.TempDir(), "missing")); err == nil { + t.Fatalf("missing root must return an error") + } +} + +func TestMeasureJobScratch(t *testing.T) { + root := testRoot(t) + if total, err := MeasureJobScratch(root); err != nil || total != 0 { + t.Fatalf("missing scratch base = %d, %v; want 0, nil", total, err) + } + writeFile(t, filepath.Join(root, JobScratchDirName, "job-aaaaaaaaaaaaaaaa", "tmp", "a"), 100) + writeFile(t, filepath.Join(root, JobScratchDirName, "job-bbbbbbbbbbbbbbbb", "go-tmp", "b"), 50) + outside := t.TempDir() + writeFile(t, filepath.Join(outside, "payload"), 9999) + linkDir := filepath.Join(root, JobScratchDirName, "job-bbbbbbbbbbbbbbbb", "tmp") + if err := os.MkdirAll(linkDir, 0o700); err != nil { + t.Fatalf("mkdir: %v", err) + } + if err := os.Symlink(filepath.Join(outside, "payload"), filepath.Join(linkDir, "link")); err != nil { + t.Fatalf("symlink: %v", err) + } + total, err := MeasureJobScratch(root) + if err != nil { + t.Fatalf("MeasureJobScratch: %v", err) + } + if total != 150 { + t.Fatalf("scratch bytes = %d, want 150", total) + } +} diff --git a/internal/commentrunner/storage/runtime_home.go b/internal/commentrunner/storage/runtime_home.go new file mode 100644 index 00000000..86ed6c4d --- /dev/null +++ b/internal/commentrunner/storage/runtime_home.go @@ -0,0 +1,316 @@ +package storage + +import ( + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "os" + "path/filepath" + "regexp" + "strings" + "time" + + "github.com/higress-group/issue-spec/internal/commentrunner/state" + "github.com/higress-group/issue-spec/internal/workspace" +) + +const ( + // RunnerHomesDirName holds the persistent runner-scoped shared runtime + // HOME roots, one per scope hash. + RunnerHomesDirName = ".runner-home" + // JobScratchDirName holds per-job disposable scratch directories. + JobScratchDirName = ".job-scratch" + + runtimeScopeFileName = "scope.json" + runtimeScopeVersion = 1 +) + +// jobScratchIDPattern is the exact scratch identity shape: the dispatcher's +// stable job IDs are "job-" plus 16 lowercase hex characters. +var jobScratchIDPattern = regexp.MustCompile(`^job-[0-9a-f]{16}$`) + +// RuntimeScope identifies the one shared runtime HOME of a runner scope: +// hostname, backend profile realm (empty for the builtin GitHub profile), +// canonical repo "owner/name", and runner identity. +type RuntimeScope struct { + Hostname string + Realm string + Repo string + Runner string +} + +// Key is the NUL-joined scope identity used as the hash preimage. NUL cannot +// appear in any field, so the join is unambiguous. +func (s RuntimeScope) Key() string { + return s.Hostname + "\x00" + s.Realm + "\x00" + s.Repo + "\x00" + s.Runner +} + +// Validate requires hostname, repo, and runner; realm may stay empty. +func (s RuntimeScope) Validate() error { + if strings.TrimSpace(s.Hostname) == "" { + return fmt.Errorf("runtime scope hostname is required") + } + if strings.TrimSpace(s.Repo) == "" { + return fmt.Errorf("runtime scope repo is required") + } + if !strings.Contains(s.Repo, "/") { + return fmt.Errorf("runtime scope repo %q must be canonical owner/name", s.Repo) + } + if strings.TrimSpace(s.Runner) == "" { + return fmt.Errorf("runtime scope runner identity is required") + } + return nil +} + +// RuntimeScopeHash is hex(sha256(scope.Key()))[:16], the same 32-char shape +// as the session runtime hash so ValidHashName accepts it. +func RuntimeScopeHash(scope RuntimeScope) (string, error) { + if err := scope.Validate(); err != nil { + return "", err + } + sum := sha256.Sum256([]byte(scope.Key())) + return hex.EncodeToString(sum[:16]), nil +} + +// RuntimeHomeRoot maps a scope to `/.runner-home/`. +// It only joins paths; PrepareRuntimeHome enforces the canonical fail-closed +// filesystem checks. +func RuntimeHomeRoot(workspaceRoot string, scope RuntimeScope) (string, error) { + hash, err := RuntimeScopeHash(scope) + if err != nil { + return "", err + } + abs, err := filepath.Abs(strings.TrimSpace(workspaceRoot)) + if err != nil { + return "", fmt.Errorf("resolve workspace root for runtime home: %w", err) + } + clean := filepath.Clean(abs) + if clean == string(os.PathSeparator) { + return "", fmt.Errorf("workspace root %q cannot be filesystem root for runtime home", clean) + } + return filepath.Join(clean, RunnerHomesDirName, hash), nil +} + +// RuntimeHomePaths is the shared runtime HOME layout: the same five subdirs +// the per-session runtime used, plus scope.json at the root. +type RuntimeHomePaths struct { + Root string + Home string + GHConfigDir string + XDGConfigHome string + CodexHome string + AcpxRuntimeDir string +} + +// RuntimeHomePathsFor derives the standard layout below a scope root. +func RuntimeHomePathsFor(root string) RuntimeHomePaths { + return RuntimeHomePaths{ + Root: root, + Home: filepath.Join(root, "home"), + GHConfigDir: filepath.Join(root, "gh"), + XDGConfigHome: filepath.Join(root, "xdg"), + CodexHome: filepath.Join(root, "codex"), + AcpxRuntimeDir: filepath.Join(root, "acpx-runtime"), + } +} + +// runtimeScopeFile is the persisted scope binding inside a runtime home root. +type runtimeScopeFile struct { + SchemaVersion int `json:"schema_version"` + Hostname string `json:"hostname"` + Realm string `json:"realm"` + Repo string `json:"repo"` + Runner string `json:"runner"` + CreatedAt time.Time `json:"created_at"` +} + +// PrepareRuntimeHome creates (or validates) the shared runtime HOME for a +// scope. Every level is created 0700 fail-closed: non-symlink directories +// only, confined below the canonical workspace root, never colliding with a +// protected root entry or an existing file. An existing scope.json must match +// the requested scope exactly; a mismatch or a foreign schema fails closed. +func PrepareRuntimeHome(workspaceRoot string, scope RuntimeScope) (RuntimeHomePaths, error) { + if err := scope.Validate(); err != nil { + return RuntimeHomePaths{}, err + } + hash, err := RuntimeScopeHash(scope) + if err != nil { + return RuntimeHomePaths{}, err + } + canonical, err := Canonicalize(workspaceRoot) + if err != nil { + return RuntimeHomePaths{}, err + } + root, err := preparePrivateLevels(canonical, RunnerHomesDirName, hash) + if err != nil { + return RuntimeHomePaths{}, err + } + paths := RuntimeHomePathsFor(root) + for _, dir := range []string{paths.Home, paths.GHConfigDir, paths.XDGConfigHome, paths.CodexHome, paths.AcpxRuntimeDir} { + if _, err := preparePrivateLevels(canonical, RunnerHomesDirName, hash, filepath.Base(dir)); err != nil { + return RuntimeHomePaths{}, err + } + } + if err := ensureRuntimeScopeFile(paths.Root, scope); err != nil { + return RuntimeHomePaths{}, err + } + return paths, nil +} + +// ensureRuntimeScopeFile writes scope.json on first prepare and pins the +// binding afterwards: an existing file must name the identical scope. +func ensureRuntimeScopeFile(root string, scope RuntimeScope) error { + path := filepath.Join(root, runtimeScopeFileName) + data, err := os.ReadFile(path) + switch { + case errors.Is(err, os.ErrNotExist): + payload, err := json.MarshalIndent(runtimeScopeFile{ + SchemaVersion: runtimeScopeVersion, + Hostname: scope.Hostname, + Realm: scope.Realm, + Repo: scope.Repo, + Runner: scope.Runner, + CreatedAt: time.Now().UTC(), + }, "", " ") + if err != nil { + return err + } + return state.WriteAtomic(path, append(payload, '\n')) + case err != nil: + return fmt.Errorf("read runtime scope file: %w", err) + } + return compareRuntimeScopeData(path, data, scope) +} + +// checkRuntimeScopeBinding validates an existing scope.json without writing: +// a missing file is not yet bound and passes. +func checkRuntimeScopeBinding(root string, scope RuntimeScope) error { + path := filepath.Join(root, runtimeScopeFileName) + data, err := os.ReadFile(path) + if errors.Is(err, os.ErrNotExist) { + return nil + } + if err != nil { + return fmt.Errorf("read runtime scope file: %w", err) + } + return compareRuntimeScopeData(path, data, scope) +} + +// compareRuntimeScopeData fails closed on an unreadable, foreign-schema, or +// scope-mismatching binding, naming both scopes in the error. +func compareRuntimeScopeData(path string, data []byte, scope RuntimeScope) error { + var existing runtimeScopeFile + if err := json.Unmarshal(data, &existing); err != nil { + return fmt.Errorf("runtime scope file %q is unreadable; refusing to reuse the home: %w", path, err) + } + if existing.SchemaVersion != runtimeScopeVersion { + return fmt.Errorf("runtime scope file %q has schema version %d, this binary requires %d", path, existing.SchemaVersion, runtimeScopeVersion) + } + if existing.Hostname != scope.Hostname || existing.Realm != scope.Realm || existing.Repo != scope.Repo || existing.Runner != scope.Runner { + return fmt.Errorf("runtime home %q is bound to scope hostname=%q realm=%q repo=%q runner=%q; refusing scope hostname=%q realm=%q repo=%q runner=%q", + path, existing.Hostname, existing.Realm, existing.Repo, existing.Runner, + scope.Hostname, scope.Realm, scope.Repo, scope.Runner) + } + return nil +} + +// JobScratchRoot maps a job ID to `/.job-scratch/`. The +// job ID must have the exact dispatcher scratch identity shape. +func JobScratchRoot(workspaceRoot, jobID string) (string, error) { + if !jobScratchIDPattern.MatchString(jobID) { + return "", fmt.Errorf("job id %q is not a valid scratch identity", jobID) + } + abs, err := filepath.Abs(strings.TrimSpace(workspaceRoot)) + if err != nil { + return "", fmt.Errorf("resolve workspace root for job scratch: %w", err) + } + clean := filepath.Clean(abs) + if clean == string(os.PathSeparator) { + return "", fmt.Errorf("workspace root %q cannot be filesystem root for job scratch", clean) + } + return filepath.Join(clean, JobScratchDirName, jobID), nil +} + +// JobScratchPaths is the per-job disposable scratch layout. +type JobScratchPaths struct { + Root string + Tmp string + GoTmp string + XDGData string + XDGState string +} + +// PrepareJobScratch creates the job's scratch tree 0700 with the same +// fail-closed checks as PrepareRuntimeHome. +func PrepareJobScratch(workspaceRoot, jobID string) (JobScratchPaths, error) { + if !jobScratchIDPattern.MatchString(jobID) { + return JobScratchPaths{}, fmt.Errorf("job id %q is not a valid scratch identity", jobID) + } + canonical, err := Canonicalize(workspaceRoot) + if err != nil { + return JobScratchPaths{}, err + } + root, err := preparePrivateLevels(canonical, JobScratchDirName, jobID) + if err != nil { + return JobScratchPaths{}, err + } + paths := JobScratchPaths{ + Root: root, + Tmp: filepath.Join(root, "tmp"), + GoTmp: filepath.Join(root, "go-tmp"), + XDGData: filepath.Join(root, "xdg-data"), + XDGState: filepath.Join(root, "xdg-state"), + } + for _, dir := range []string{paths.Tmp, paths.GoTmp, paths.XDGData, paths.XDGState} { + if _, err := preparePrivateLevels(canonical, JobScratchDirName, jobID, filepath.Base(dir)); err != nil { + return JobScratchPaths{}, err + } + } + return paths, nil +} + +// preparePrivateLevels creates each component below the canonical root like +// the dispatcher's preparePrivateCanonicalDir: 0700, Lstat-verified +// non-symlink directories, and a final canonical confinement proof. +func preparePrivateLevels(canonicalRoot string, components ...string) (string, error) { + current := canonicalRoot + for _, component := range components { + if protectedRootEntries[component] { + return "", fmt.Errorf("entry %q collides with a protected root entry", component) + } + current = filepath.Join(current, component) + info, err := os.Lstat(current) + switch { + case errors.Is(err, os.ErrNotExist): + if err := os.Mkdir(current, 0o700); err != nil && !errors.Is(err, os.ErrExist) { + return "", fmt.Errorf("create private directory %q: %w", current, err) + } + info, err = os.Lstat(current) + if err != nil { + return "", fmt.Errorf("verify private directory %q: %w", current, err) + } + case err != nil: + return "", fmt.Errorf("inspect private directory %q: %w", current, err) + } + if info.Mode()&os.ModeSymlink != 0 || !info.IsDir() { + return "", fmt.Errorf("path %q must be a non-symlink directory", current) + } + if err := os.Chmod(current, 0o700); err != nil { + return "", fmt.Errorf("enforce private permissions on %q: %w", current, err) + } + } + resolved, err := filepath.EvalSymlinks(current) + if err != nil || filepath.Clean(resolved) != current { + return "", fmt.Errorf("path %q must be canonical without symlink traversal", current) + } + confined, err := workspace.ValidatePathUnderRoot(canonicalRoot, current) + if err != nil { + return "", err + } + if confined != current { + return "", fmt.Errorf("path %q escapes the canonical workspace root", current) + } + return current, nil +} diff --git a/internal/commentrunner/storage/runtime_home_test.go b/internal/commentrunner/storage/runtime_home_test.go new file mode 100644 index 00000000..031d1163 --- /dev/null +++ b/internal/commentrunner/storage/runtime_home_test.go @@ -0,0 +1,270 @@ +package storage + +import ( + "encoding/json" + "os" + "path/filepath" + "strings" + "testing" +) + +func testScope() RuntimeScope { + return RuntimeScope{Hostname: "host-1", Realm: "", Repo: "o/r", Runner: "runner-1"} +} + +func TestRuntimeScopeHashDeterministicShape(t *testing.T) { + first, err := RuntimeScopeHash(testScope()) + if err != nil { + t.Fatalf("RuntimeScopeHash: %v", err) + } + second, err := RuntimeScopeHash(testScope()) + if err != nil { + t.Fatalf("RuntimeScopeHash: %v", err) + } + if first != second { + t.Fatalf("hash not deterministic: %q vs %q", first, second) + } + if !ValidHashName(first) { + t.Fatalf("hash %q does not have the physical hash shape", first) + } +} + +func TestRuntimeScopeHashVariesPerField(t *testing.T) { + base, err := RuntimeScopeHash(testScope()) + if err != nil { + t.Fatalf("RuntimeScopeHash: %v", err) + } + variants := map[string]RuntimeScope{ + "hostname": {Hostname: "host-2", Repo: "o/r", Runner: "runner-1"}, + "realm": {Hostname: "host-1", Realm: "enterprise", Repo: "o/r", Runner: "runner-1"}, + "repo": {Hostname: "host-1", Repo: "o/r2", Runner: "runner-1"}, + "runner": {Hostname: "host-1", Repo: "o/r", Runner: "runner-2"}, + } + for field, scope := range variants { + hash, err := RuntimeScopeHash(scope) + if err != nil { + t.Fatalf("RuntimeScopeHash %s: %v", field, err) + } + if hash == base { + t.Fatalf("changing %s must change the scope hash", field) + } + } +} + +func TestRuntimeScopeValidate(t *testing.T) { + cases := map[string]RuntimeScope{ + "valid": testScope(), + "valid with realm": {Hostname: "h", Realm: "ent", Repo: "o/r", Runner: "r"}, + "missing hostname": {Repo: "o/r", Runner: "r"}, + "missing repo": {Hostname: "h", Runner: "r"}, + "repo not canonical": {Hostname: "h", Repo: "repo-only", Runner: "r"}, + "missing runner": {Hostname: "h", Repo: "o/r"}, + "whitespace only": {Hostname: " ", Repo: "o/r", Runner: "r"}, + } + for name, scope := range cases { + err := scope.Validate() + switch { + case strings.HasPrefix(name, "valid") && err != nil: + t.Fatalf("%s: unexpected error %v", name, err) + case !strings.HasPrefix(name, "valid") && err == nil: + t.Fatalf("%s: expected validation error", name) + } + } +} + +func TestRuntimeHomeRootLayout(t *testing.T) { + root := testRoot(t) + scope := testScope() + homeRoot, err := RuntimeHomeRoot(root, scope) + if err != nil { + t.Fatalf("RuntimeHomeRoot: %v", err) + } + if filepath.Dir(homeRoot) != filepath.Join(root, RunnerHomesDirName) { + t.Fatalf("runtime home root %q is not below %s", homeRoot, RunnerHomesDirName) + } + if !ValidHashName(filepath.Base(homeRoot)) { + t.Fatalf("runtime home base %q is not a scope hash", filepath.Base(homeRoot)) + } +} + +func TestPrepareRuntimeHomeCreatesPrivateTree(t *testing.T) { + root := testRoot(t) + scope := testScope() + paths, err := PrepareRuntimeHome(root, scope) + if err != nil { + t.Fatalf("PrepareRuntimeHome: %v", err) + } + for _, dir := range []string{paths.Root, paths.Home, paths.GHConfigDir, paths.XDGConfigHome, paths.CodexHome, paths.AcpxRuntimeDir} { + info, err := os.Lstat(dir) + if err != nil { + t.Fatalf("lstat %q: %v", dir, err) + } + if info.Mode()&os.ModeSymlink != 0 || !info.IsDir() { + t.Fatalf("%q is not a non-symlink directory", dir) + } + if info.Mode().Perm() != 0o700 { + t.Fatalf("%q perm = %o, want 0700", dir, info.Mode().Perm()) + } + } + data, err := os.ReadFile(filepath.Join(paths.Root, runtimeScopeFileName)) + if err != nil { + t.Fatalf("read scope file: %v", err) + } + var binding runtimeScopeFile + if err := json.Unmarshal(data, &binding); err != nil { + t.Fatalf("parse scope file: %v", err) + } + if binding.SchemaVersion != runtimeScopeVersion || binding.Hostname != scope.Hostname || + binding.Realm != scope.Realm || binding.Repo != scope.Repo || binding.Runner != scope.Runner { + t.Fatalf("scope binding = %+v, want scope %+v", binding, scope) + } + if binding.CreatedAt.IsZero() { + t.Fatalf("scope binding created_at must be set") + } + // Idempotent: a second prepare validates and preserves the original binding. + if _, err := PrepareRuntimeHome(root, scope); err != nil { + t.Fatalf("second PrepareRuntimeHome: %v", err) + } + again, err := os.ReadFile(filepath.Join(paths.Root, runtimeScopeFileName)) + if err != nil { + t.Fatalf("re-read scope file: %v", err) + } + if string(again) != string(data) { + t.Fatalf("second prepare rewrote scope.json") + } +} + +func TestPrepareRuntimeHomeScopeMismatchFails(t *testing.T) { + root := testRoot(t) + scope := testScope() + paths, err := PrepareRuntimeHome(root, scope) + if err != nil { + t.Fatalf("PrepareRuntimeHome: %v", err) + } + // Simulate a foreign binding landing at this scope hash (collision/tamper). + foreign := runtimeScopeFile{ + SchemaVersion: runtimeScopeVersion, + Hostname: scope.Hostname, + Repo: "other/repo", + Runner: scope.Runner, + } + payload, err := json.Marshal(foreign) + if err != nil { + t.Fatalf("marshal: %v", err) + } + if err := os.WriteFile(filepath.Join(paths.Root, runtimeScopeFileName), payload, 0o600); err != nil { + t.Fatalf("write foreign scope: %v", err) + } + _, err = PrepareRuntimeHome(root, scope) + if err == nil { + t.Fatalf("scope mismatch must fail closed") + } + if !strings.Contains(err.Error(), "other/repo") || !strings.Contains(err.Error(), "o/r") { + t.Fatalf("mismatch error must name both scopes, got: %v", err) + } +} + +func TestPrepareRuntimeHomeRejectsSymlink(t *testing.T) { + root := testRoot(t) + scope := testScope() + outside := t.TempDir() + if err := os.Symlink(outside, filepath.Join(root, RunnerHomesDirName)); err != nil { + t.Fatalf("symlink base: %v", err) + } + if _, err := PrepareRuntimeHome(root, scope); err == nil { + t.Fatalf("symlinked %s must fail closed", RunnerHomesDirName) + } +} + +func TestPrepareRuntimeHomeRejectsSymlinkedScopeDir(t *testing.T) { + root := testRoot(t) + scope := testScope() + hash, err := RuntimeScopeHash(scope) + if err != nil { + t.Fatalf("hash: %v", err) + } + base := filepath.Join(root, RunnerHomesDirName) + if err := os.MkdirAll(base, 0o700); err != nil { + t.Fatalf("mkdir base: %v", err) + } + outside := t.TempDir() + if err := os.Symlink(outside, filepath.Join(base, hash)); err != nil { + t.Fatalf("symlink scope dir: %v", err) + } + if _, err := PrepareRuntimeHome(root, scope); err == nil { + t.Fatalf("symlinked scope directory must fail closed") + } +} + +func TestPrepareRuntimeHomeRejectsFileCollision(t *testing.T) { + root := testRoot(t) + if err := os.WriteFile(filepath.Join(root, RunnerHomesDirName), []byte("x"), 0o600); err != nil { + t.Fatalf("write colliding file: %v", err) + } + if _, err := PrepareRuntimeHome(root, testScope()); err == nil { + t.Fatalf("an existing file at %s must fail closed", RunnerHomesDirName) + } +} + +func TestJobScratchRootValidation(t *testing.T) { + root := testRoot(t) + valid := "job-0123456789abcdef" + path, err := JobScratchRoot(root, valid) + if err != nil { + t.Fatalf("JobScratchRoot: %v", err) + } + if path != filepath.Join(root, JobScratchDirName, valid) { + t.Fatalf("scratch root %q unexpected", path) + } + invalid := []string{ + "", + "job-1", + "job-0123456789abcde", // too short + "job-0123456789abcdef0", // too long + "job-0123456789abcdeg", // non-hex + "JOB-0123456789abcdef", // wrong prefix case + " job-0123456789abcdef", // surrounding space + "job-0123456789abcdef/extra", + } + for _, id := range invalid { + if _, err := JobScratchRoot(root, id); err == nil { + t.Fatalf("job id %q must be rejected", id) + } + if _, err := PrepareJobScratch(root, id); err == nil { + t.Fatalf("PrepareJobScratch job id %q must be rejected", id) + } + } +} + +func TestPrepareJobScratchLayout(t *testing.T) { + root := testRoot(t) + jobID := "job-aaaaaaaaaaaaaaaa" + paths, err := PrepareJobScratch(root, jobID) + if err != nil { + t.Fatalf("PrepareJobScratch: %v", err) + } + for _, dir := range []string{paths.Root, paths.Tmp, paths.GoTmp, paths.XDGData, paths.XDGState} { + info, err := os.Lstat(dir) + if err != nil { + t.Fatalf("lstat %q: %v", dir, err) + } + if info.Mode()&os.ModeSymlink != 0 || !info.IsDir() || info.Mode().Perm() != 0o700 { + t.Fatalf("%q must be a 0700 non-symlink directory, mode=%v", dir, info.Mode()) + } + } + // Idempotent re-prepare. + if _, err := PrepareJobScratch(root, jobID); err != nil { + t.Fatalf("second PrepareJobScratch: %v", err) + } +} + +func TestPrepareJobScratchRejectsSymlink(t *testing.T) { + root := testRoot(t) + outside := t.TempDir() + if err := os.Symlink(outside, filepath.Join(root, JobScratchDirName)); err != nil { + t.Fatalf("symlink base: %v", err) + } + if _, err := PrepareJobScratch(root, "job-bbbbbbbbbbbbbbbb"); err == nil { + t.Fatalf("symlinked %s must fail closed", JobScratchDirName) + } +} diff --git a/internal/commentrunner/storage/runtime_reconcile.go b/internal/commentrunner/storage/runtime_reconcile.go new file mode 100644 index 00000000..5921ec10 --- /dev/null +++ b/internal/commentrunner/storage/runtime_reconcile.go @@ -0,0 +1,365 @@ +package storage + +import ( + "context" + "errors" + "fmt" + "os" + "path/filepath" + "sort" + "strings" + + "github.com/higress-group/issue-spec/internal/commentrunner/state" + "github.com/higress-group/issue-spec/internal/workspace" +) + +// RuntimeReconcileReport summarizes one job-scratch reconciliation or cache +// eviction pass. Outcome lists record applied mutations only; a dry run +// reports its would-be actions through Diagnostics and never mutates. +type RuntimeReconcileReport struct { + ScratchRemoved []string `json:"scratch_removed,omitempty"` + ScratchKept []string `json:"scratch_kept,omitempty"` + ScratchRejected []string `json:"scratch_rejected,omitempty"` + CacheEvicted []string `json:"cache_evicted,omitempty"` + ReclaimedBytes int64 `json:"reclaimed_bytes,omitempty"` + Diagnostics []string `json:"diagnostics,omitempty"` +} + +// jobScratchActive mirrors buildProtectionView's active job statuses: an +// interrupted job still counts as active because its sandbox may be reaped +// asynchronously, so its scratch is never reclaimed underneath it. +func jobScratchActive(status state.LifecycleStatus) bool { + switch status { + case state.StatusQueued, state.StatusDispatched, state.StatusRunning, state.StatusInterrupted: + return true + default: + return false + } +} + +// validateScratchDeletionTarget is the scratch counterpart of +// ValidateDeletionTarget: exact job identity, literal path below +// `.job-scratch`, confined to the canonical root, existing non-symlink +// directory. Missing targets validate so removals stay idempotent. +func validateScratchDeletionTarget(workspaceRoot, target, jobID string) error { + if !jobScratchIDPattern.MatchString(jobID) { + return fmt.Errorf("deletion target job id %q is invalid", jobID) + } + if strings.TrimSpace(target) == "" { + return errors.New("deletion target is required") + } + clean := filepath.Clean(target) + canonicalRoot, err := Canonicalize(workspaceRoot) + if err != nil { + return err + } + if clean == canonicalRoot { + return fmt.Errorf("deletion target %q is the workspace root", target) + } + parent, base := filepath.Dir(clean), filepath.Base(clean) + if base != jobID { + return fmt.Errorf("deletion target %q does not match job id %q", target, jobID) + } + if parent != filepath.Join(canonicalRoot, JobScratchDirName) { + return fmt.Errorf("deletion target %q is outside the job scratch directory", target) + } + confined, err := workspace.ValidatePathUnderRoot(canonicalRoot, clean) + if err != nil { + return err + } + if confined != clean { + return fmt.Errorf("deletion target %q escapes its literal path", target) + } + info, err := os.Lstat(clean) + if errors.Is(err, os.ErrNotExist) { + return nil + } + if err != nil { + return err + } + if info.Mode()&os.ModeSymlink != 0 { + return fmt.Errorf("deletion target %q is a symlink", target) + } + if !info.IsDir() { + return fmt.Errorf("deletion target %q is not a directory", target) + } + return nil +} + +// ReconcileJobScratch removes scratch of terminal or unknown jobs and keeps +// scratch of active jobs, both for recorded entries and for on-disk leftovers +// a crashed runner never recorded. Foreign names below `.job-scratch` are +// rejected and never deleted. The pass is idempotent: a second apply is a +// no-op. +func (s *Service) ReconcileJobScratch(ctx context.Context, apply bool) (RuntimeReconcileReport, error) { + _, release, err := EnsureOwner(ctx, s.root) + if err != nil { + return RuntimeReconcileReport{}, fmt.Errorf("job scratch reconcile owner: %w", err) + } + defer release() + st, err := s.stateLoader(ctx) + if err != nil { + return RuntimeReconcileReport{}, fmt.Errorf("load runner state for job scratch reconciliation: %w", err) + } + st.Normalize() + active := map[string]bool{} + for _, job := range st.Jobs { + if jobScratchActive(job.Status) { + active[job.ID] = true + } + } + store, err := s.RuntimeStore() + if err != nil { + return RuntimeReconcileReport{}, err + } + if err := store.Reload(); err != nil { + return RuntimeReconcileReport{}, fmt.Errorf("reload runtime metadata: %w", err) + } + report := RuntimeReconcileReport{} + mutate := apply + if store.Status() == SidecarReportOnly { + mutate = false + report.Diagnostics = append(report.Diagnostics, "runtime metadata is report-only: foreign root identity or newer schema; inventory only, no mutations") + } + if cause := store.LoadCause(); cause != nil { + report.Diagnostics = append(report.Diagnostics, safeDiagnostic("runtime metadata rebuilt after corruption ("+cause.Err.Error()+")")) + } + if !apply { + report.Diagnostics = append(report.Diagnostics, "dry-run: no mutations performed") + } + + recorded := store.State().Scratch + recordedIDs := make([]string, 0, len(recorded)) + for jobID := range recorded { + recordedIDs = append(recordedIDs, jobID) + } + sort.Strings(recordedIDs) + for _, jobID := range recordedIDs { + record := recorded[jobID] + if active[jobID] { + report.ScratchKept = append(report.ScratchKept, jobID) + if mutate && record.CleanupState != CleanupManaged { + // A crash-interrupted completion raced a still-active job: + // heal the record instead of deleting live scratch. + if err := store.Update(func(st *RuntimeState) error { + current, ok := st.Scratch[jobID] + if ok { + current.CleanupState = CleanupManaged + st.Scratch[jobID] = current + } + return nil + }); err != nil { + report.Diagnostics = append(report.Diagnostics, safeDiagnostic("job scratch "+jobID+" heal failed: "+err.Error())) + } + } + continue + } + if _, statErr := os.Lstat(record.Path); errors.Is(statErr, os.ErrNotExist) { + // Directory already gone: garbage-collect the record. + if mutate { + if err := store.Update(func(st *RuntimeState) error { + delete(st.Scratch, jobID) + return nil + }); err != nil { + report.Diagnostics = append(report.Diagnostics, safeDiagnostic("job scratch "+jobID+" record GC failed: "+err.Error())) + } + } + continue + } + if err := validateScratchDeletionTarget(s.root, record.Path, jobID); err != nil { + report.Diagnostics = append(report.Diagnostics, safeDiagnostic("job scratch "+jobID+" record fails validation: "+err.Error())) + continue + } + if !mutate { + report.Diagnostics = append(report.Diagnostics, "would remove job scratch "+jobID) + continue + } + if err := s.removeJobScratch(store, record, &report); err != nil { + report.Diagnostics = append(report.Diagnostics, safeDiagnostic("job scratch "+jobID+" removal failed: "+err.Error())) + } + } + + // On-disk entries with no metadata record. + names, err := readDirNames(filepath.Join(s.root, JobScratchDirName)) + if err == nil { + for _, name := range names { + if _, ok := recorded[name]; ok { + continue + } + if !jobScratchIDPattern.MatchString(name) { + report.ScratchRejected = append(report.ScratchRejected, name) + report.Diagnostics = append(report.Diagnostics, safeDiagnostic("job scratch entry "+name+" is not a scratch identity; rejected, never deleted")) + continue + } + if active[name] { + report.ScratchKept = append(report.ScratchKept, name) + continue + } + path := filepath.Join(s.root, JobScratchDirName, name) + if err := validateScratchDeletionTarget(s.root, path, name); err != nil { + report.Diagnostics = append(report.Diagnostics, safeDiagnostic("job scratch entry "+name+" fails validation: "+err.Error())) + continue + } + if !mutate { + report.Diagnostics = append(report.Diagnostics, "would remove unrecorded job scratch "+name) + continue + } + measured := measureTreeBytes(path) + if err := removeOpenedTree(path, nil); err != nil { + report.Diagnostics = append(report.Diagnostics, safeDiagnostic("job scratch entry "+name+" removal failed: "+err.Error())) + continue + } + report.ScratchRemoved = append(report.ScratchRemoved, name) + report.ReclaimedBytes += measured + } + } + sort.Strings(report.ScratchRemoved) + sort.Strings(report.ScratchKept) + sort.Strings(report.ScratchRejected) + return report, nil +} + +// removeJobScratch runs the recorded-entry lifecycle: mark deleting, remove +// the directory capability scoped, then garbage-collect the record. +func (s *Service) removeJobScratch(store *RuntimeStore, record JobScratchRecord, report *RuntimeReconcileReport) error { + if err := store.Update(func(st *RuntimeState) error { + current, ok := st.Scratch[record.JobID] + if !ok { + return nil + } + current.CleanupState = CleanupDeleting + st.Scratch[record.JobID] = current + return nil + }); err != nil { + return err + } + measured := measureTreeBytes(record.Path) + if err := removeOpenedTree(record.Path, nil); err != nil { + return err + } + if err := store.Update(func(st *RuntimeState) error { + delete(st.Scratch, record.JobID) + return nil + }); err != nil { + return err + } + report.ScratchRemoved = append(report.ScratchRemoved, record.JobID) + report.ReclaimedBytes += measured + return nil +} + +// EvictRuntimeCaches removes only the rebuildable cache subtrees of every +// recorded runtime home, in eviction priority order. Protected identity and +// configuration paths are never deletion targets. +func (s *Service) EvictRuntimeCaches(ctx context.Context, apply bool) (RuntimeReconcileReport, error) { + _, release, err := EnsureOwner(ctx, s.root) + if err != nil { + return RuntimeReconcileReport{}, fmt.Errorf("runtime cache eviction owner: %w", err) + } + defer release() + store, err := s.RuntimeStore() + if err != nil { + return RuntimeReconcileReport{}, err + } + if err := store.Reload(); err != nil { + return RuntimeReconcileReport{}, fmt.Errorf("reload runtime metadata: %w", err) + } + report := RuntimeReconcileReport{} + mutate := apply + if store.Status() == SidecarReportOnly { + mutate = false + report.Diagnostics = append(report.Diagnostics, "runtime metadata is report-only: foreign root identity or newer schema; inventory only, no mutations") + } + homes := store.State().Homes + hashes := make([]string, 0, len(homes)) + for hash := range homes { + hashes = append(hashes, hash) + } + sort.Strings(hashes) + for _, hash := range hashes { + home := homes[hash] + if err := s.validateRuntimeHomeRecord(home); err != nil { + report.Diagnostics = append(report.Diagnostics, safeDiagnostic("runtime home "+hash+" record fails validation: "+err.Error())) + continue + } + homeDir := RuntimeHomePathsFor(home.Path).Home + for _, cacheDir := range RuntimeCacheDirs(homeDir) { + info, statErr := os.Lstat(cacheDir) + if errors.Is(statErr, os.ErrNotExist) { + continue + } + if statErr != nil { + report.Diagnostics = append(report.Diagnostics, safeDiagnostic("runtime cache "+cacheDir+" lstat failed: "+statErr.Error())) + continue + } + if err := validateCacheEvictionTarget(homeDir, cacheDir, info); err != nil { + report.Diagnostics = append(report.Diagnostics, safeDiagnostic("runtime cache "+cacheDir+" fails validation: "+err.Error())) + continue + } + measured := measureTreeBytes(cacheDir) + if !mutate { + report.Diagnostics = append(report.Diagnostics, fmt.Sprintf("would evict runtime cache %s (%d bytes)", cacheDir, measured)) + continue + } + if err := removeOpenedTree(cacheDir, nil); err != nil { + report.Diagnostics = append(report.Diagnostics, safeDiagnostic("runtime cache "+cacheDir+" removal failed: "+err.Error())) + continue + } + report.CacheEvicted = append(report.CacheEvicted, cacheDir) + report.ReclaimedBytes += measured + } + } + sort.Strings(report.CacheEvicted) + return report, nil +} + +// validateRuntimeHomeRecord proves a recorded home still names the canonical +// scoped path below this root before any of its subtrees are touched. +func (s *Service) validateRuntimeHomeRecord(home RuntimeHomeRecord) error { + if !ValidHashName(home.Hash) { + return fmt.Errorf("home hash %q is invalid", home.Hash) + } + expected := filepath.Join(s.root, RunnerHomesDirName, home.Hash) + if filepath.Clean(home.Path) != expected { + return fmt.Errorf("home path %q does not match the scoped home %q for this root", home.Path, expected) + } + info, err := os.Lstat(home.Path) + if err != nil { + return err + } + if info.Mode()&os.ModeSymlink != 0 || !info.IsDir() { + return fmt.Errorf("home path %q must be a non-symlink directory", home.Path) + } + return nil +} + +// validateCacheEvictionTarget performs the final identity check on one cache +// directory: it must be one of the exact eviction-eligible paths of this +// home, confined below the literal home path, and a non-symlink directory. +func validateCacheEvictionTarget(home, target string, info os.FileInfo) error { + clean := filepath.Clean(target) + eligible := false + for _, candidate := range RuntimeCacheDirs(home) { + if candidate == clean { + eligible = true + break + } + } + if !eligible { + return fmt.Errorf("eviction target %q is not an eligible cache directory", target) + } + confined, err := workspace.ValidatePathUnderRoot(home, clean) + if err != nil { + return err + } + if confined != clean { + return fmt.Errorf("eviction target %q escapes its literal path", target) + } + if info.Mode()&os.ModeSymlink != 0 { + return fmt.Errorf("eviction target %q is a symlink", target) + } + if !info.IsDir() { + return fmt.Errorf("eviction target %q is not a directory", target) + } + return nil +} diff --git a/internal/commentrunner/storage/runtime_reconcile_test.go b/internal/commentrunner/storage/runtime_reconcile_test.go new file mode 100644 index 00000000..1cf78210 --- /dev/null +++ b/internal/commentrunner/storage/runtime_reconcile_test.go @@ -0,0 +1,387 @@ +package storage + +import ( + "context" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/higress-group/issue-spec/internal/commentrunner/state" +) + +const ( + scratchJobActive = "job-aaaaaaaaaaaaaaaa" + scratchJobDone = "job-bbbbbbbbbbbbbbbb" + scratchJobOrphan = "job-cccccccccccccccc" + scratchJobDryRun = "job-dddddddddddddddd" +) + +func newRuntimeService(t *testing.T, st state.RunnerState) (*Service, string) { + t.Helper() + root := testRoot(t) + svc, err := NewService(ServiceConfig{ + WorkspaceRoot: root, + StateLoader: func(context.Context) (state.RunnerState, error) { return st, nil }, + OrphanGrace: DefaultOrphanGrace, + }) + if err != nil { + t.Fatalf("NewService: %v", err) + } + t.Cleanup(func() { _ = svc.Close() }) + return svc, root +} + +func prepareScratchWithFile(t *testing.T, root, jobID string, size int) JobScratchPaths { + t.Helper() + paths, err := PrepareJobScratch(root, jobID) + if err != nil { + t.Fatalf("PrepareJobScratch: %v", err) + } + writeFile(t, filepath.Join(paths.Tmp, "payload"), size) + return paths +} + +func contains(values []string, needle string) bool { + for _, value := range values { + if value == needle { + return true + } + } + return false +} + +func TestReconcileJobScratchKeepsActiveJob(t *testing.T) { + st := state.NewState() + st.Jobs[scratchJobActive] = state.Job{ID: scratchJobActive, Repo: "o/r", Status: state.StatusRunning} + svc, root := newRuntimeService(t, st) + paths := prepareScratchWithFile(t, root, scratchJobActive, 128) + if err := svc.RecordJobScratch(context.Background(), scratchJobActive, paths.Root); err != nil { + t.Fatalf("RecordJobScratch: %v", err) + } + report, err := svc.ReconcileJobScratch(context.Background(), true) + if err != nil { + t.Fatalf("ReconcileJobScratch: %v", err) + } + if !contains(report.ScratchKept, scratchJobActive) { + t.Fatalf("active job scratch must be kept: %+v", report) + } + if len(report.ScratchRemoved) != 0 { + t.Fatalf("active job scratch must not be removed: %+v", report) + } + if _, err := os.Lstat(paths.Root); err != nil { + t.Fatalf("active job scratch dir removed: %v", err) + } + store, err := svc.RuntimeStore() + if err != nil { + t.Fatalf("RuntimeStore: %v", err) + } + if record := store.State().Scratch[scratchJobActive]; record.CleanupState != CleanupManaged { + t.Fatalf("record cleanup state = %q, want managed", record.CleanupState) + } + // Interrupted jobs are still active for scratch purposes. + st.Jobs[scratchJobActive] = state.Job{ID: scratchJobActive, Repo: "o/r", Status: state.StatusInterrupted} + report, err = svc.ReconcileJobScratch(context.Background(), true) + if err != nil { + t.Fatalf("ReconcileJobScratch interrupted: %v", err) + } + if !contains(report.ScratchKept, scratchJobActive) { + t.Fatalf("interrupted job scratch must be kept: %+v", report) + } +} + +func TestReconcileJobScratchRemovesTerminalAndUnknown(t *testing.T) { + st := state.NewState() + st.Jobs[scratchJobDone] = state.Job{ID: scratchJobDone, Repo: "o/r", Status: state.StatusCompleted} + svc, root := newRuntimeService(t, st) + donePaths := prepareScratchWithFile(t, root, scratchJobDone, 100) + orphanPaths := prepareScratchWithFile(t, root, scratchJobOrphan, 50) + for jobID, paths := range map[string]JobScratchPaths{scratchJobDone: donePaths, scratchJobOrphan: orphanPaths} { + if err := svc.RecordJobScratch(context.Background(), jobID, paths.Root); err != nil { + t.Fatalf("RecordJobScratch %s: %v", jobID, err) + } + } + report, err := svc.ReconcileJobScratch(context.Background(), true) + if err != nil { + t.Fatalf("ReconcileJobScratch: %v", err) + } + if !contains(report.ScratchRemoved, scratchJobDone) || !contains(report.ScratchRemoved, scratchJobOrphan) { + t.Fatalf("terminal and unknown job scratch must be removed: %+v", report) + } + if report.ReclaimedBytes != 150 { + t.Fatalf("reclaimed = %d, want 150", report.ReclaimedBytes) + } + for _, paths := range []JobScratchPaths{donePaths, orphanPaths} { + if _, err := os.Lstat(paths.Root); !os.IsNotExist(err) { + t.Fatalf("scratch dir %q must be gone, err=%v", paths.Root, err) + } + } + store, err := svc.RuntimeStore() + if err != nil { + t.Fatalf("RuntimeStore: %v", err) + } + if len(store.State().Scratch) != 0 { + t.Fatalf("scratch records must be garbage-collected: %+v", store.State().Scratch) + } + // Second apply is a no-op. + second, err := svc.ReconcileJobScratch(context.Background(), true) + if err != nil { + t.Fatalf("second ReconcileJobScratch: %v", err) + } + if len(second.ScratchRemoved) != 0 || second.ReclaimedBytes != 0 { + t.Fatalf("second pass must be a no-op: %+v", second) + } +} + +func TestReconcileJobScratchRemovesUnrecordedOnDiskEntry(t *testing.T) { + svc, root := newRuntimeService(t, state.NewState()) + // A crashed runner left scratch behind without a metadata record. + paths := prepareScratchWithFile(t, root, scratchJobOrphan, 64) + report, err := svc.ReconcileJobScratch(context.Background(), true) + if err != nil { + t.Fatalf("ReconcileJobScratch: %v", err) + } + if !contains(report.ScratchRemoved, scratchJobOrphan) { + t.Fatalf("unrecorded on-disk scratch must be removed: %+v", report) + } + if _, err := os.Lstat(paths.Root); !os.IsNotExist(err) { + t.Fatalf("scratch dir must be gone, err=%v", err) + } +} + +func TestReconcileJobScratchRejectsForeignNames(t *testing.T) { + svc, root := newRuntimeService(t, state.NewState()) + for _, name := range []string{"random-junk", "job-ZZZZZZZZZZZZZZZZ", "job-1234"} { + if err := os.MkdirAll(filepath.Join(root, JobScratchDirName, name), 0o700); err != nil { + t.Fatalf("mkdir: %v", err) + } + } + report, err := svc.ReconcileJobScratch(context.Background(), true) + if err != nil { + t.Fatalf("ReconcileJobScratch: %v", err) + } + for _, name := range []string{"random-junk", "job-ZZZZZZZZZZZZZZZZ", "job-1234"} { + if !contains(report.ScratchRejected, name) { + t.Fatalf("foreign entry %q must be rejected: %+v", name, report) + } + if _, err := os.Lstat(filepath.Join(root, JobScratchDirName, name)); err != nil { + t.Fatalf("foreign entry %q must never be deleted: %v", name, err) + } + } + if len(report.ScratchRemoved) != 0 { + t.Fatalf("no scratch may be removed: %+v", report) + } +} + +func TestReconcileJobScratchDryRunDoesNotMutate(t *testing.T) { + st := state.NewState() + st.Jobs[scratchJobDryRun] = state.Job{ID: scratchJobDryRun, Repo: "o/r", Status: state.StatusFailed} + svc, root := newRuntimeService(t, st) + paths := prepareScratchWithFile(t, root, scratchJobDryRun, 32) + if err := svc.RecordJobScratch(context.Background(), scratchJobDryRun, paths.Root); err != nil { + t.Fatalf("RecordJobScratch: %v", err) + } + report, err := svc.ReconcileJobScratch(context.Background(), false) + if err != nil { + t.Fatalf("ReconcileJobScratch dry-run: %v", err) + } + if len(report.ScratchRemoved) != 0 || report.ReclaimedBytes != 0 { + t.Fatalf("dry-run must not report removals: %+v", report) + } + if !contains(report.Diagnostics, "would remove job scratch "+scratchJobDryRun) { + t.Fatalf("dry-run must report the would-delete: %+v", report.Diagnostics) + } + if _, err := os.Lstat(paths.Root); err != nil { + t.Fatalf("dry-run must not delete: %v", err) + } + store, err := svc.RuntimeStore() + if err != nil { + t.Fatalf("RuntimeStore: %v", err) + } + if record := store.State().Scratch[scratchJobDryRun]; record.CleanupState != CleanupManaged { + t.Fatalf("dry-run must not mutate records, state = %q", record.CleanupState) + } +} + +func TestCompleteJobScratch(t *testing.T) { + svc, root := newRuntimeService(t, state.NewState()) + paths := prepareScratchWithFile(t, root, scratchJobDone, 16) + if err := svc.RecordJobScratch(context.Background(), scratchJobDone, paths.Root); err != nil { + t.Fatalf("RecordJobScratch: %v", err) + } + if err := svc.CompleteJobScratch(context.Background(), scratchJobDone); err != nil { + t.Fatalf("CompleteJobScratch: %v", err) + } + if _, err := os.Lstat(paths.Root); !os.IsNotExist(err) { + t.Fatalf("scratch dir must be gone, err=%v", err) + } + store, err := svc.RuntimeStore() + if err != nil { + t.Fatalf("RuntimeStore: %v", err) + } + if len(store.State().Scratch) != 0 { + t.Fatalf("record must be garbage-collected: %+v", store.State().Scratch) + } + // Idempotent and tolerant of unknown jobs. + if err := svc.CompleteJobScratch(context.Background(), scratchJobDone); err != nil { + t.Fatalf("second CompleteJobScratch: %v", err) + } + if err := svc.CompleteJobScratch(context.Background(), scratchJobOrphan); err != nil { + t.Fatalf("unknown job must be a no-op: %v", err) + } + if err := svc.CompleteJobScratch(context.Background(), "job-1"); err == nil { + t.Fatalf("invalid job id must be rejected") + } +} + +func TestRecordJobScratchRejectsForeignPath(t *testing.T) { + svc, root := newRuntimeService(t, state.NewState()) + if err := svc.RecordJobScratch(context.Background(), scratchJobDone, filepath.Join(root, "elsewhere", scratchJobDone)); err == nil { + t.Fatalf("recording a scratch path outside .job-scratch must fail closed") + } + if err := svc.RecordJobScratch(context.Background(), "job-1", filepath.Join(root, JobScratchDirName, "job-1")); err == nil { + t.Fatalf("invalid job id must be rejected") + } +} + +func TestEvictRuntimeCaches(t *testing.T) { + svc, root := newRuntimeService(t, state.NewState()) + scope := testScope() + paths, err := PrepareRuntimeHome(root, scope) + if err != nil { + t.Fatalf("PrepareRuntimeHome: %v", err) + } + if err := svc.RecordRuntimeHome(context.Background(), scope, paths); err != nil { + t.Fatalf("RecordRuntimeHome: %v", err) + } + writeFile(t, filepath.Join(paths.Home, ".npm", "_npx", "pkg", "i.js"), 10) + writeFile(t, filepath.Join(paths.Home, ".npm", "registry.tgz"), 20) + writeFile(t, filepath.Join(paths.Home, ".cache", "go-build", "abc"), 30) + writeFile(t, filepath.Join(paths.Home, "go", "pkg", "mod", "m.zip"), 40) + protectedFiles := []string{ + filepath.Join(paths.Home, ".acpx", "sessions", "index.json"), + filepath.Join(paths.Home, ".qoder", "settings.json"), + filepath.Join(paths.Home, ".claude.json"), + filepath.Join(paths.GHConfigDir, "hosts.yml"), + filepath.Join(paths.CodexHome, "sessions", "s1.json"), + filepath.Join(paths.Root, "scope.json"), + } + for _, path := range protectedFiles[:5] { + writeFile(t, path, 64) + } + + dry, err := svc.EvictRuntimeCaches(context.Background(), false) + if err != nil { + t.Fatalf("EvictRuntimeCaches dry-run: %v", err) + } + if len(dry.CacheEvicted) != 0 || dry.ReclaimedBytes != 0 { + t.Fatalf("dry-run must not evict: %+v", dry) + } + for _, dir := range RuntimeCacheDirs(paths.Home) { + if _, err := os.Lstat(dir); err != nil { + t.Fatalf("dry-run removed %q: %v", dir, err) + } + } + + report, err := svc.EvictRuntimeCaches(context.Background(), true) + if err != nil { + t.Fatalf("EvictRuntimeCaches: %v", err) + } + if len(report.CacheEvicted) != 4 { + t.Fatalf("evicted = %v, want 4 cache dirs", report.CacheEvicted) + } + if report.ReclaimedBytes != 100 { + t.Fatalf("reclaimed = %d, want 100", report.ReclaimedBytes) + } + for _, dir := range RuntimeCacheDirs(paths.Home) { + if _, err := os.Lstat(dir); !os.IsNotExist(err) { + t.Fatalf("cache dir %q must be gone, err=%v", dir, err) + } + } + for _, path := range protectedFiles { + if _, err := os.Lstat(path); err != nil { + t.Fatalf("protected path %q must survive eviction: %v", path, err) + } + } + // Second apply is a no-op. + second, err := svc.EvictRuntimeCaches(context.Background(), true) + if err != nil { + t.Fatalf("second EvictRuntimeCaches: %v", err) + } + if len(second.CacheEvicted) != 0 || second.ReclaimedBytes != 0 { + t.Fatalf("second eviction must be a no-op: %+v", second) + } +} + +func TestEvictRuntimeCachesNeverFollowsSymlinks(t *testing.T) { + svc, root := newRuntimeService(t, state.NewState()) + scope := testScope() + paths, err := PrepareRuntimeHome(root, scope) + if err != nil { + t.Fatalf("PrepareRuntimeHome: %v", err) + } + if err := svc.RecordRuntimeHome(context.Background(), scope, paths); err != nil { + t.Fatalf("RecordRuntimeHome: %v", err) + } + outside := t.TempDir() + writeFile(t, filepath.Join(outside, "payload"), 99) + if err := os.Symlink(outside, filepath.Join(paths.Home, ".cache")); err != nil { + t.Fatalf("symlink: %v", err) + } + report, err := svc.EvictRuntimeCaches(context.Background(), true) + if err != nil { + t.Fatalf("EvictRuntimeCaches: %v", err) + } + if contains(report.CacheEvicted, filepath.Join(paths.Home, ".cache")) { + t.Fatalf("a symlinked cache dir must never be evicted: %+v", report) + } + if len(report.Diagnostics) == 0 { + t.Fatalf("symlinked cache dir must produce a diagnostic") + } + info, err := os.Lstat(filepath.Join(paths.Home, ".cache")) + if err != nil || info.Mode()&os.ModeSymlink == 0 { + t.Fatalf("symlink must remain, info=%v err=%v", info, err) + } + if _, err := os.Lstat(filepath.Join(outside, "payload")); err != nil { + t.Fatalf("symlink target must be untouched: %v", err) + } +} + +func TestEvictRuntimeCachesRejectsForeignHomePath(t *testing.T) { + svc, root := newRuntimeService(t, state.NewState()) + scope := testScope() + paths, err := PrepareRuntimeHome(root, scope) + if err != nil { + t.Fatalf("PrepareRuntimeHome: %v", err) + } + if err := svc.RecordRuntimeHome(context.Background(), scope, paths); err != nil { + t.Fatalf("RecordRuntimeHome: %v", err) + } + // Tamper: repoint the recorded home outside the root. + store, err := svc.RuntimeStore() + if err != nil { + t.Fatalf("RuntimeStore: %v", err) + } + hash, err := RuntimeScopeHash(scope) + if err != nil { + t.Fatalf("hash: %v", err) + } + if err := store.Update(func(st *RuntimeState) error { + record := st.Homes[hash] + record.Path = filepath.Join(root, ".sessions", strings.Repeat("de", 16)) + st.Homes[hash] = record + return nil + }); err != nil { + t.Fatalf("tamper update: %v", err) + } + report, err := svc.EvictRuntimeCaches(context.Background(), true) + if err != nil { + t.Fatalf("EvictRuntimeCaches: %v", err) + } + if len(report.CacheEvicted) != 0 { + t.Fatalf("a home outside the managed root must never be evicted: %+v", report) + } + if len(report.Diagnostics) == 0 { + t.Fatalf("tampered home record must produce a diagnostic") + } +} diff --git a/internal/commentrunner/storage/runtime_store.go b/internal/commentrunner/storage/runtime_store.go new file mode 100644 index 00000000..be17e5da --- /dev/null +++ b/internal/commentrunner/storage/runtime_store.go @@ -0,0 +1,387 @@ +package storage + +import ( + "bytes" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "strings" + "sync" + "time" + + "github.com/higress-group/issue-spec/internal/commentrunner/state" +) + +// RuntimeSchemaVersion is the current `.storage/runtime.json` schema. A newer +// on-disk schema or a foreign root identity permits report-only inventory, +// exactly like the sidecar. The runtime metadata lives in its own file so a +// v1 binary never parses records it cannot classify. +const RuntimeSchemaVersion = 1 + +const ( + runtimeFileName = "runtime.json" + runtimeLockName = "runtime.json.lock" + runtimeCorruptBackup = "runtime-corrupt-latest.json" +) + +// MigrationState is the ledger lifecycle of one scope's legacy-home import: +// imported -> validated -> retired. Transitions are monotonic. +type MigrationState string + +const ( + MigrationImported MigrationState = "imported" + MigrationValidated MigrationState = "validated" + MigrationRetired MigrationState = "retired" +) + +func (s MigrationState) Valid() bool { + switch s { + case MigrationImported, MigrationValidated, MigrationRetired: + return true + default: + return false + } +} + +// migrationStateRank orders ledger states; regressions are rejected. +func migrationStateRank(s MigrationState) int { + switch s { + case MigrationImported: + return 1 + case MigrationValidated: + return 2 + case MigrationRetired: + return 3 + default: + return 0 + } +} + +// RuntimeHomeRecord pins one scope hash to its prepared runtime home. +type RuntimeHomeRecord struct { + Hash string `json:"hash"` + Path string `json:"path"` + Hostname string `json:"hostname"` + Realm string `json:"realm,omitempty"` + Repo string `json:"repo"` + Runner string `json:"runner"` + CreatedAt time.Time `json:"created_at,omitempty"` +} + +// JobScratchRecord tracks one job's scratch directory through the same +// managed/deleting/removed cleanup lifecycle as sidecar resources. +type JobScratchRecord struct { + JobID string `json:"job_id"` + Path string `json:"path"` + CreatedAt time.Time `json:"created_at,omitempty"` + CleanupState CleanupState `json:"cleanup_state,omitempty"` +} + +// MigrationRecord is the per-scope legacy-home migration ledger entry. +type MigrationRecord struct { + ScopeHash string `json:"scope_hash"` + State MigrationState `json:"state"` + ImportedSessions []string `json:"imported_sessions,omitempty"` + ValidatedSession string `json:"validated_session,omitempty"` + UpdatedAt time.Time `json:"updated_at,omitempty"` +} + +// RuntimeState is the `.storage/runtime.json` document. +type RuntimeState struct { + SchemaVersion int `json:"schema_version"` + RootIdentity string `json:"root_identity"` + Homes map[string]RuntimeHomeRecord `json:"homes,omitempty"` + Scratch map[string]JobScratchRecord `json:"scratch,omitempty"` + Migrations map[string]MigrationRecord `json:"migrations,omitempty"` + UpdatedAt time.Time `json:"updated_at,omitempty"` +} + +// NewRuntimeState builds an empty runtime metadata state bound to the root. +func NewRuntimeState(rootIdentity string) RuntimeState { + return RuntimeState{ + SchemaVersion: RuntimeSchemaVersion, + RootIdentity: rootIdentity, + Homes: map[string]RuntimeHomeRecord{}, + Scratch: map[string]JobScratchRecord{}, + Migrations: map[string]MigrationRecord{}, + } +} + +// RuntimeStore is the locked, atomically persisted `.storage/runtime.json` +// metadata store. It mirrors the sidecar Store semantics: the runtime flock is +// a leaf-level lock held only for short load/mutate/save sections, missing or +// corrupt files open rebuilt, and foreign-root or newer-schema files open +// report-only. Corruption causes reuse the sidecar's CorruptSidecarError so +// callers handle one evidence-loss shape. +type RuntimeStore struct { + mu sync.Mutex + dir string + path string + lockPath string + root string + identity string + state RuntimeState + status SidecarStatus + closed bool + now func() time.Time + backupError error + loadCause *CorruptSidecarError +} + +// OpenRuntimeStore loads the runtime metadata for the canonical workspace +// root, creating the private `.storage` directory when needed. +func OpenRuntimeStore(workspaceRoot string) (*RuntimeStore, error) { + root := strings.TrimSpace(workspaceRoot) + if root == "" { + return nil, fmt.Errorf("workspace root is required") + } + canonical, err := Canonicalize(root) + if err != nil { + return nil, err + } + identity, err := RootIdentity(canonical) + if err != nil { + return nil, err + } + dir := filepath.Join(canonical, StorageDirName) + if err := os.MkdirAll(dir, 0o700); err != nil { + return nil, fmt.Errorf("prepare storage directory: %w", err) + } + store := &RuntimeStore{ + dir: dir, + path: filepath.Join(dir, runtimeFileName), + lockPath: filepath.Join(dir, runtimeLockName), + root: canonical, + identity: identity, + now: func() time.Time { return time.Now().UTC() }, + } + if err := store.load(); err != nil { + return nil, err + } + return store, nil +} + +// Status reports how much trust this process may place in the loaded state. +func (s *RuntimeStore) Status() SidecarStatus { return s.status } + +// RootIdentity returns the identity hash bound to this store. +func (s *RuntimeStore) RootIdentity() string { return s.identity } + +// Path returns the runtime metadata file path. +func (s *RuntimeStore) Path() string { return s.path } + +// BackupError reports a non-fatal failure to preserve a corrupt file. +func (s *RuntimeStore) BackupError() error { return s.backupError } + +// LoadCause reports why the state was rebuilt from corrupt bytes, if it was. +func (s *RuntimeStore) LoadCause() *CorruptSidecarError { return s.loadCause } + +// Reload re-reads the runtime metadata under its lock so a long-lived store +// observes writes made by other stores on the same root. +func (s *RuntimeStore) Reload() error { + s.mu.Lock() + defer s.mu.Unlock() + if s.closed { + return fmt.Errorf("runtime metadata store is closed") + } + unlock, err := s.lock() + if err != nil { + return err + } + defer unlock() + return s.load() +} + +// State returns a copy of the currently loaded runtime state. +func (s *RuntimeStore) State() RuntimeState { + s.mu.Lock() + defer s.mu.Unlock() + return cloneRuntimeState(s.state) +} + +func (s *RuntimeStore) load() error { + s.state = NewRuntimeState(s.identity) + s.loadCause = nil + s.backupError = nil + data, err := os.ReadFile(s.path) + switch { + case errors.Is(err, os.ErrNotExist): + s.status = SidecarRebuilt + return nil + case err != nil: + return fmt.Errorf("read runtime metadata: %w", err) + } + if len(bytes.TrimSpace(data)) == 0 { + return s.rebuildFromCorrupt(fmt.Errorf("empty runtime metadata")) + } + var decoded RuntimeState + dec := json.NewDecoder(bytes.NewReader(data)) + if err := dec.Decode(&decoded); err != nil { + return s.rebuildFromCorrupt(err) + } + var trailing any + if err := dec.Decode(&trailing); err != io.EOF { + return s.rebuildFromCorrupt(fmt.Errorf("unexpected trailing JSON")) + } + if decoded.SchemaVersion > RuntimeSchemaVersion { + s.status = SidecarReportOnly + s.state = normalizeRuntimeState(decoded) + return nil + } + if decoded.RootIdentity != "" && decoded.RootIdentity != s.identity { + s.status = SidecarReportOnly + s.state = normalizeRuntimeState(decoded) + return nil + } + decoded = normalizeRuntimeState(decoded) + for id, record := range decoded.Scratch { + if !record.CleanupState.Valid() { + return s.rebuildFromCorrupt(fmt.Errorf("scratch record %q has invalid cleanup state", id)) + } + } + for hash, record := range decoded.Migrations { + if !record.State.Valid() { + return s.rebuildFromCorrupt(fmt.Errorf("migration record %q has invalid state", hash)) + } + } + s.state = decoded + s.status = SidecarReady + return nil +} + +func (s *RuntimeStore) rebuildFromCorrupt(cause error) error { + s.state = NewRuntimeState(s.identity) + s.status = SidecarRebuilt + s.loadCause = &CorruptSidecarError{Path: s.path, Err: cause} + if _, err := s.backupCorruptCopy(); err != nil { + s.backupError = err + } + return nil +} + +func (s *RuntimeStore) backupCorruptCopy() (string, error) { + data, err := os.ReadFile(s.path) + if err != nil || len(bytes.TrimSpace(data)) == 0 { + return "", err + } + backupDir := filepath.Join(s.dir, backupDirName) + if err := os.MkdirAll(backupDir, 0o700); err != nil { + return "", err + } + // Stable name: a persistently corrupt file must not grow backups + // unboundedly across passes. + backup := filepath.Join(backupDir, runtimeCorruptBackup) + if err := state.WriteAtomic(backup, data); err != nil { + return "", err + } + return backup, nil +} + +// Update runs one locked read-modify-write cycle on the runtime metadata. It +// fails on report-only stores so a foreign or newer file is never rewritten. +func (s *RuntimeStore) Update(mutate func(*RuntimeState) error) error { + if mutate == nil { + return fmt.Errorf("runtime metadata update callback is required") + } + s.mu.Lock() + defer s.mu.Unlock() + if s.closed { + return fmt.Errorf("runtime metadata store is closed") + } + if s.status == SidecarReportOnly { + return ErrReportOnly + } + unlock, err := s.lock() + if err != nil { + return err + } + defer unlock() + // Re-load inside the lock so interleaved writers never lose updates. + if err := s.load(); err != nil { + return err + } + if s.status == SidecarReportOnly { + return ErrReportOnly + } + next := cloneRuntimeState(s.state) + if err := mutate(&next); err != nil { + return err + } + if next.SchemaVersion != RuntimeSchemaVersion { + return fmt.Errorf("runtime metadata schema version must stay %d", RuntimeSchemaVersion) + } + if next.RootIdentity != s.identity { + return fmt.Errorf("runtime metadata root identity must stay bound to this root") + } + next = normalizeRuntimeState(next) + next.UpdatedAt = s.now() + data, err := json.MarshalIndent(next, "", " ") + if err != nil { + return err + } + data = append(data, '\n') + if err := state.WriteAtomic(s.path, data); err != nil { + return err + } + s.state = next + s.status = SidecarReady + return nil +} + +func (s *RuntimeStore) lock() (func(), error) { + file, err := os.OpenFile(s.lockPath, os.O_RDWR|os.O_CREATE, 0o600) + if err != nil { + return nil, fmt.Errorf("open runtime metadata lock: %w", err) + } + if err := flockExclusive(file); err != nil { + _ = file.Close() + return nil, fmt.Errorf("lock runtime metadata: %w", err) + } + return func() { + _ = flockUnlock(file) + _ = file.Close() + }, nil +} + +// Close releases the store handle; the flock itself is never held between +// operations, so close only gates further use. +func (s *RuntimeStore) Close() error { + s.mu.Lock() + defer s.mu.Unlock() + s.closed = true + return nil +} + +func normalizeRuntimeState(st RuntimeState) RuntimeState { + if st.Homes == nil { + st.Homes = map[string]RuntimeHomeRecord{} + } + if st.Scratch == nil { + st.Scratch = map[string]JobScratchRecord{} + } + if st.Migrations == nil { + st.Migrations = map[string]MigrationRecord{} + } + return st +} + +func cloneRuntimeState(st RuntimeState) RuntimeState { + clone := st + clone.Homes = make(map[string]RuntimeHomeRecord, len(st.Homes)) + for hash, record := range st.Homes { + clone.Homes[hash] = record + } + clone.Scratch = make(map[string]JobScratchRecord, len(st.Scratch)) + for id, record := range st.Scratch { + clone.Scratch[id] = record + } + clone.Migrations = make(map[string]MigrationRecord, len(st.Migrations)) + for hash, record := range st.Migrations { + record.ImportedSessions = append([]string(nil), record.ImportedSessions...) + clone.Migrations[hash] = record + } + return clone +} diff --git a/internal/commentrunner/storage/runtime_store_test.go b/internal/commentrunner/storage/runtime_store_test.go new file mode 100644 index 00000000..05079023 --- /dev/null +++ b/internal/commentrunner/storage/runtime_store_test.go @@ -0,0 +1,238 @@ +package storage + +import ( + "errors" + "os" + "path/filepath" + "strings" + "testing" + "time" +) + +func TestOpenRuntimeStoreMissingRebuilds(t *testing.T) { + root := testRoot(t) + store, err := OpenRuntimeStore(root) + if err != nil { + t.Fatalf("OpenRuntimeStore: %v", err) + } + defer store.Close() + if store.Status() != SidecarRebuilt { + t.Fatalf("status = %q, want %q", store.Status(), SidecarRebuilt) + } + st := store.State() + if st.SchemaVersion != RuntimeSchemaVersion { + t.Fatalf("schema = %d, want %d", st.SchemaVersion, RuntimeSchemaVersion) + } + if st.RootIdentity != wantIdentity(t, root) { + t.Fatalf("root identity = %q, want %q", st.RootIdentity, wantIdentity(t, root)) + } + if len(st.Homes) != 0 || len(st.Scratch) != 0 || len(st.Migrations) != 0 { + t.Fatalf("fresh state must have empty maps: %+v", st) + } +} + +func TestRuntimeStoreUpdatePersistsAtomically(t *testing.T) { + root := testRoot(t) + store, err := OpenRuntimeStore(root) + if err != nil { + t.Fatalf("OpenRuntimeStore: %v", err) + } + home := RuntimeHomeRecord{ + Hash: strings.Repeat("ab", 16), Path: filepath.Join(root, RunnerHomesDirName, strings.Repeat("ab", 16)), + Hostname: "host-1", Repo: "o/r", Runner: "runner-1", CreatedAt: time.Now().UTC(), + } + if err := store.Update(func(st *RuntimeState) error { + st.Homes[home.Hash] = home + return nil + }); err != nil { + t.Fatalf("Update: %v", err) + } + if store.Status() != SidecarReady { + t.Fatalf("status after update = %q, want %q", store.Status(), SidecarReady) + } + if err := store.Close(); err != nil { + t.Fatalf("Close: %v", err) + } + reopened, err := OpenRuntimeStore(root) + if err != nil { + t.Fatalf("reopen: %v", err) + } + defer reopened.Close() + if reopened.Status() != SidecarReady { + t.Fatalf("reopened status = %q, want %q", reopened.Status(), SidecarReady) + } + got, ok := reopened.State().Homes[home.Hash] + if !ok || got != home { + t.Fatalf("home record = %+v ok=%v, want %+v", got, ok, home) + } +} + +func TestRuntimeStoreUpdatePinsSchemaAndRoot(t *testing.T) { + root := testRoot(t) + store, err := OpenRuntimeStore(root) + if err != nil { + t.Fatalf("OpenRuntimeStore: %v", err) + } + defer store.Close() + if err := store.Update(func(st *RuntimeState) error { + st.SchemaVersion = RuntimeSchemaVersion + 1 + return nil + }); err == nil { + t.Fatalf("schema bump must be rejected") + } + if err := store.Update(func(st *RuntimeState) error { + st.RootIdentity = strings.Repeat("0", 64) + return nil + }); err == nil { + t.Fatalf("root identity change must be rejected") + } +} + +func TestRuntimeStoreCorruptBackupAndRebuild(t *testing.T) { + root := testRoot(t) + store, err := OpenRuntimeStore(root) + if err != nil { + t.Fatalf("OpenRuntimeStore: %v", err) + } + if err := store.Update(func(st *RuntimeState) error { + st.Scratch["job-aaaaaaaaaaaaaaaa"] = JobScratchRecord{JobID: "job-aaaaaaaaaaaaaaaa", CleanupState: CleanupManaged} + return nil + }); err != nil { + t.Fatalf("Update: %v", err) + } + path := store.Path() + if err := store.Close(); err != nil { + t.Fatalf("Close: %v", err) + } + corrupt := []byte("{not json") + if err := os.WriteFile(path, corrupt, 0o600); err != nil { + t.Fatalf("corrupt: %v", err) + } + rebuilt, err := OpenRuntimeStore(root) + if err != nil { + t.Fatalf("reopen corrupt: %v", err) + } + defer rebuilt.Close() + if rebuilt.Status() != SidecarRebuilt { + t.Fatalf("status = %q, want %q", rebuilt.Status(), SidecarRebuilt) + } + if rebuilt.LoadCause() == nil { + t.Fatalf("load cause must explain the rebuild") + } + backup := filepath.Join(root, StorageDirName, backupDirName, runtimeCorruptBackup) + data, err := os.ReadFile(backup) + if err != nil { + t.Fatalf("corrupt backup missing: %v", err) + } + if string(data) != string(corrupt) { + t.Fatalf("corrupt backup content mismatch") + } + if len(rebuilt.State().Scratch) != 0 { + t.Fatalf("rebuilt state must drop corrupt records") + } +} + +func TestRuntimeStoreNewerSchemaReportOnly(t *testing.T) { + root := testRoot(t) + path := filepath.Join(root, StorageDirName, runtimeFileName) + payload := []byte(`{"schema_version":2,"root_identity":"` + wantIdentity(t, root) + `"}`) + if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil { + t.Fatalf("mkdir: %v", err) + } + if err := os.WriteFile(path, payload, 0o600); err != nil { + t.Fatalf("write: %v", err) + } + store, err := OpenRuntimeStore(root) + if err != nil { + t.Fatalf("OpenRuntimeStore: %v", err) + } + defer store.Close() + if store.Status() != SidecarReportOnly { + t.Fatalf("status = %q, want %q", store.Status(), SidecarReportOnly) + } + if store.State().SchemaVersion != 2 { + t.Fatalf("report-only state must expose the newer schema for inventory") + } + if err := store.Update(func(st *RuntimeState) error { return nil }); !errors.Is(err, ErrReportOnly) { + t.Fatalf("Update err = %v, want ErrReportOnly", err) + } +} + +func TestRuntimeStoreForeignRootReportOnly(t *testing.T) { + root := testRoot(t) + path := filepath.Join(root, StorageDirName, runtimeFileName) + payload := []byte(`{"schema_version":1,"root_identity":"` + strings.Repeat("0", 64) + `"}`) + if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil { + t.Fatalf("mkdir: %v", err) + } + if err := os.WriteFile(path, payload, 0o600); err != nil { + t.Fatalf("write: %v", err) + } + store, err := OpenRuntimeStore(root) + if err != nil { + t.Fatalf("OpenRuntimeStore: %v", err) + } + defer store.Close() + if store.Status() != SidecarReportOnly { + t.Fatalf("status = %q, want %q", store.Status(), SidecarReportOnly) + } + if err := store.Update(func(st *RuntimeState) error { return nil }); !errors.Is(err, ErrReportOnly) { + t.Fatalf("Update err = %v, want ErrReportOnly", err) + } +} + +func TestRuntimeStoreSequentialWritersPreserved(t *testing.T) { + root := testRoot(t) + first, err := OpenRuntimeStore(root) + if err != nil { + t.Fatalf("open first: %v", err) + } + defer first.Close() + second, err := OpenRuntimeStore(root) + if err != nil { + t.Fatalf("open second: %v", err) + } + defer second.Close() + if err := first.Update(func(st *RuntimeState) error { + st.Homes["home-1"] = RuntimeHomeRecord{Hash: "home-1"} + return nil + }); err != nil { + t.Fatalf("first update: %v", err) + } + if err := second.Update(func(st *RuntimeState) error { + st.Scratch["job-cccccccccccccccc"] = JobScratchRecord{JobID: "job-cccccccccccccccc", CleanupState: CleanupManaged} + return nil + }); err != nil { + t.Fatalf("second update: %v", err) + } + if err := first.Reload(); err != nil { + t.Fatalf("reload: %v", err) + } + st := first.State() + if _, ok := st.Homes["home-1"]; !ok { + t.Fatalf("first writer's home record lost: %+v", st.Homes) + } + if _, ok := st.Scratch["job-cccccccccccccccc"]; !ok { + t.Fatalf("second writer's scratch record lost: %+v", st.Scratch) + } +} + +func TestRuntimeStoreInvalidCleanupStateRebuilds(t *testing.T) { + root := testRoot(t) + path := filepath.Join(root, StorageDirName, runtimeFileName) + payload := []byte(`{"schema_version":1,"root_identity":"` + wantIdentity(t, root) + `","scratch":{"job-x":{"job_id":"job-x","cleanup_state":"bogus"}}}`) + if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil { + t.Fatalf("mkdir: %v", err) + } + if err := os.WriteFile(path, payload, 0o600); err != nil { + t.Fatalf("write: %v", err) + } + store, err := OpenRuntimeStore(root) + if err != nil { + t.Fatalf("OpenRuntimeStore: %v", err) + } + defer store.Close() + if store.Status() != SidecarRebuilt { + t.Fatalf("status = %q, want %q for invalid cleanup state", store.Status(), SidecarRebuilt) + } +} diff --git a/internal/commentrunner/storage/service.go b/internal/commentrunner/storage/service.go index b5398f4c..8626353d 100644 --- a/internal/commentrunner/storage/service.go +++ b/internal/commentrunner/storage/service.go @@ -6,6 +6,7 @@ import ( "fmt" "os" "path/filepath" + "sort" "strings" "sync" "time" @@ -45,6 +46,7 @@ type Service struct { mu sync.Mutex root string store *Store + runtime *RuntimeStore stateLoader StateLoader poolInspector PoolInspector poolRemover PoolRemover @@ -121,8 +123,34 @@ func (s *Service) Root() string { return s.root } // Store exposes the sidecar for lifecycle tests and close management. func (s *Service) Store() *Store { return s.store } -// Close releases the sidecar. -func (s *Service) Close() error { return s.store.Close() } +// RuntimeStore returns the lazily opened `.storage/runtime.json` metadata +// store, shared by every entry point of this logical run. +func (s *Service) RuntimeStore() (*RuntimeStore, error) { + s.mu.Lock() + defer s.mu.Unlock() + if s.runtime != nil { + return s.runtime, nil + } + store, err := OpenRuntimeStore(s.root) + if err != nil { + return nil, err + } + s.runtime = store + return store, nil +} + +// Close releases the sidecar and the runtime metadata store. +func (s *Service) Close() error { + s.mu.Lock() + runtime := s.runtime + s.mu.Unlock() + if runtime != nil { + if err := runtime.Close(); err != nil { + return err + } + } + return s.store.Close() +} // AdmitDispatch performs statfs-only minimum-free-space admission before any // session/workspace lock is acquired. Below threshold it runs one locked safe @@ -239,6 +267,196 @@ func (s *Service) RecordSessionResources(_ context.Context, repo, publicSessionI return nil } +// RecordRuntimeHome upserts the prepared runtime home of one scope before the +// home is exposed to sandbox execution. It is fail-closed: the recorded root +// must be the scope's canonical home below this root, and an upsert failure +// means the home would be unmanaged. +func (s *Service) RecordRuntimeHome(_ context.Context, scope RuntimeScope, paths RuntimeHomePaths) error { + if err := scope.Validate(); err != nil { + return err + } + hash, err := RuntimeScopeHash(scope) + if err != nil { + return err + } + expected := filepath.Join(s.root, RunnerHomesDirName, hash) + if filepath.Clean(paths.Root) != expected { + return fmt.Errorf("runtime home path %q does not match the scoped home %q for this root", paths.Root, expected) + } + store, err := s.RuntimeStore() + if err != nil { + return err + } + now := s.now().UTC() + desired := func(existing RuntimeHomeRecord) RuntimeHomeRecord { + record := existing + if record.Hash == "" { + record = RuntimeHomeRecord{Hash: hash, CreatedAt: now} + } + record.Path = expected + record.Hostname = scope.Hostname + record.Realm = scope.Realm + record.Repo = scope.Repo + record.Runner = scope.Runner + return record + } + // Skip the write when the record already matches: steady-state touches + // must not fsync the metadata store per dispatched job. + if current := store.State().Homes[hash]; current == desired(current) { + return nil + } + if err := store.Update(func(st *RuntimeState) error { + st.Homes[hash] = desired(st.Homes[hash]) + return nil + }); err != nil { + return fmt.Errorf("record runtime home: %w", err) + } + return nil +} + +// RecordJobScratch upserts one job's scratch directory as managed. The path +// must be the job's canonical scratch root below this root. +func (s *Service) RecordJobScratch(_ context.Context, jobID, path string) error { + if !jobScratchIDPattern.MatchString(jobID) { + return fmt.Errorf("job id %q is not a valid scratch identity", jobID) + } + expected := filepath.Join(s.root, JobScratchDirName, jobID) + if filepath.Clean(path) != expected { + return fmt.Errorf("job scratch path %q does not match the job scratch root %q for this root", path, expected) + } + store, err := s.RuntimeStore() + if err != nil { + return err + } + now := s.now().UTC() + desired := func(existing JobScratchRecord) JobScratchRecord { + record := existing + if record.JobID == "" { + record = JobScratchRecord{JobID: jobID, CreatedAt: now} + } + record.Path = expected + record.CleanupState = CleanupManaged + return record + } + if current := store.State().Scratch[jobID]; current == desired(current) { + return nil + } + if err := store.Update(func(st *RuntimeState) error { + st.Scratch[jobID] = desired(st.Scratch[jobID]) + return nil + }); err != nil { + return fmt.Errorf("record job scratch: %w", err) + } + return nil +} + +// CompleteJobScratch removes one terminal job's scratch through the +// recoverable lifecycle: mark deleting, remove the directory capability +// scoped, then garbage-collect the record. A missing directory completes +// idempotently and an unknown job ID is a no-op. +func (s *Service) CompleteJobScratch(_ context.Context, jobID string) error { + if !jobScratchIDPattern.MatchString(jobID) { + return fmt.Errorf("job id %q is not a valid scratch identity", jobID) + } + store, err := s.RuntimeStore() + if err != nil { + return err + } + record, ok := store.State().Scratch[jobID] + if !ok { + return nil + } + if err := validateScratchDeletionTarget(s.root, record.Path, jobID); err != nil { + return fmt.Errorf("complete job scratch %q: %w", jobID, err) + } + if err := store.Update(func(st *RuntimeState) error { + current, ok := st.Scratch[jobID] + if !ok { + return nil + } + current.CleanupState = CleanupDeleting + st.Scratch[jobID] = current + return nil + }); err != nil { + return fmt.Errorf("mark job scratch %q deleting: %w", jobID, err) + } + if err := removeOpenedTree(record.Path, nil); err != nil { + return fmt.Errorf("remove job scratch %q: %w", jobID, err) + } + // The directory is gone: drop the record instead of leaving a removed + // tombstone, since job scratch has no orphan-observation lifecycle. + if err := store.Update(func(st *RuntimeState) error { + delete(st.Scratch, jobID) + return nil + }); err != nil { + return fmt.Errorf("finalize job scratch %q: %w", jobID, err) + } + return nil +} + +// RuntimeMigrationLedger returns the migration ledger entry for one scope +// hash, if present. +func (s *Service) RuntimeMigrationLedger(_ context.Context, scopeHash string) (MigrationRecord, bool, error) { + if !ValidHashName(scopeHash) { + return MigrationRecord{}, false, fmt.Errorf("scope hash %q is not a complete scope hash", scopeHash) + } + store, err := s.RuntimeStore() + if err != nil { + return MigrationRecord{}, false, err + } + record, ok := store.State().Migrations[scopeHash] + return record, ok, nil +} + +// MarkRuntimeMigration upserts a migration ledger entry. Ledger states are +// monotonic (imported < validated < retired): regressions are rejected, and +// ImportedSessions merge as a union so concurrent resume validations never +// shrink the proof set. +func (s *Service) MarkRuntimeMigration(_ context.Context, record MigrationRecord) error { + if !ValidHashName(record.ScopeHash) { + return fmt.Errorf("scope hash %q is not a complete scope hash", record.ScopeHash) + } + if !record.State.Valid() { + return fmt.Errorf("migration state %q is invalid", record.State) + } + store, err := s.RuntimeStore() + if err != nil { + return err + } + if err := store.Update(func(st *RuntimeState) error { + existing, ok := st.Migrations[record.ScopeHash] + if ok && migrationStateRank(record.State) < migrationStateRank(existing.State) { + return fmt.Errorf("migration ledger for scope %q cannot regress from %q to %q", record.ScopeHash, existing.State, record.State) + } + merged := record + if ok { + merged.ImportedSessions = unionStrings(existing.ImportedSessions, record.ImportedSessions) + if merged.ValidatedSession == "" { + merged.ValidatedSession = existing.ValidatedSession + } + } + st.Migrations[record.ScopeHash] = merged + return nil + }); err != nil { + return fmt.Errorf("mark runtime migration: %w", err) + } + return nil +} + +func unionStrings(left, right []string) []string { + seen := map[string]bool{} + var merged []string + for _, value := range append(append([]string(nil), left...), right...) { + if value == "" || seen[value] { + continue + } + seen[value] = true + merged = append(merged, value) + } + sort.Strings(merged) + return merged +} + // ReconcileStorage runs the shared engine under the canonical root owner: // every destructive storage pass proves ownership first, so an admission // cleanup inside a standalone dispatch can never bypass the owner lock. From 844ea1cd4c689492c88a6959d6c2a28148acc38a Mon Sep 17 00:00:00 2001 From: johnlanni Date: Sat, 8 Aug 2026 21:33:49 +0800 Subject: [PATCH 03/17] Revert "docs(specs): materialize durable specs for runner-scoped shared HOME" This reverts commit ae58638b6cb51d30a88019f3d3d37213954abdff. Signed-off-by: johnlanni --- .../acpx-comment-triggered-workflow/spec.md | 4 +- .../specs/runner-coordinator-agents/spec.md | 9 +- .../specs/runner-shared-runtime-home/spec.md | 103 ------------------ 3 files changed, 4 insertions(+), 112 deletions(-) delete mode 100644 issue-spec/specs/runner-shared-runtime-home/spec.md diff --git a/issue-spec/specs/acpx-comment-triggered-workflow/spec.md b/issue-spec/specs/acpx-comment-triggered-workflow/spec.md index da230d4a..0b0a2cdf 100644 --- a/issue-spec/specs/acpx-comment-triggered-workflow/spec.md +++ b/issue-spec/specs/acpx-comment-triggered-workflow/spec.md @@ -124,7 +124,7 @@ The runner MUST invoke acpx as an external coordinator backend using argv arrays #### Scenario: default sandbox - **WHEN** a runner job dispatches on Linux without `--unsafe-no-sandbox` -- **THEN** acpx SHALL run through bubblewrap with workspace filesystem isolation, the persistent runner-scoped isolated runtime `HOME` shared by that runner scope's public sessions (never the operator's real HOME) with managed `GH_CONFIG_DIR`, `XDG_CONFIG_HOME`, and `CODEX_HOME` when needed, unique disposable per-job scratch directories for `TMPDIR`, `GOTMPDIR`, `XDG_DATA_HOME`, and `XDG_STATE_HOME`, inherited proxy settings, broad token environment variables scrubbed, required system paths read-only, and the managed workspace mounted for writes. +- **THEN** acpx SHALL run through bubblewrap with workspace filesystem isolation, temporary `HOME`, `GH_CONFIG_DIR`, `XDG_CONFIG_HOME`, and `CODEX_HOME` when needed, inherited proxy settings, broad token environment variables scrubbed, required system paths read-only, and the managed workspace mounted for writes. #### Scenario: unsafe mode @@ -152,7 +152,7 @@ The runner MUST invoke acpx as an external coordinator backend using argv arrays - **THEN** the runner SHALL treat acpx as the top-level headless coordinator transport and persist bounded child provenance reported by the coordinator without needing direct access to the native worker runtime. Source SPEC comments: -- https://github.com/higress-group/issue-spec/issues/439#issuecomment-5226066261 +- https://github.com/higress-group/issue-spec/issues/343#issuecomment-5099614411 ### Requirement: durable job state and recovery diff --git a/issue-spec/specs/runner-coordinator-agents/spec.md b/issue-spec/specs/runner-coordinator-agents/spec.md index a690576e..6169295b 100644 --- a/issue-spec/specs/runner-coordinator-agents/spec.md +++ b/issue-spec/specs/runner-coordinator-agents/spec.md @@ -38,7 +38,7 @@ Source SPEC comments: ### Requirement: Runner mirrors host qoder configuration into the sandbox -The runner MUST mirror the host ~/.qoder/settings.json and the regular files directly inside ~/.qoder/.auth into the sandbox runtime HOME (the persistent runner-scoped isolated HOME exposed to the sandbox as $HOME, referred to below as $TempHome). The .auth mirror MUST be one level only, MUST preserve restrictive source permission bits (defaulting to 0600), MUST remove stale mirrored files, and MUST NOT follow destination symlinks or mirror cache, log, session-state, symlink, directory, or device entries. Because the runtime HOME is shared by concurrent jobs of one runner scope, each mirrored file refresh MUST be atomic and idempotent so a concurrent job never observes a partially written configuration or credential file. +The runner MUST mirror the host ~/.qoder/settings.json and the regular files directly inside ~/.qoder/.auth into the sandbox temporary HOME. The .auth mirror MUST be one level only, MUST preserve restrictive source permission bits (defaulting to 0600), MUST remove stale mirrored files, and MUST NOT follow destination symlinks or mirror cache, log, session-state, symlink, directory, or device entries. #### Scenario: settings.json is mirrored into sandbox TempHome @@ -65,13 +65,8 @@ The runner MUST mirror the host ~/.qoder/settings.json and the regular files dir - **WHEN** the host qoder config contains symlinks, subdirectories, devices, cache, logs, or session state - **THEN** the mirror MUST skip those entries, MUST NOT recurse, and MUST NOT expose them inside the sandbox -#### Scenario: concurrent dispatches refresh the shared mirror safely - -- **WHEN** two jobs of the same runner scope dispatch concurrently and both refresh the qoder mirror in the shared runtime HOME -- **THEN** each mirrored file MUST be replaced atomically so a running agent observes either the previous or the new complete file, never a partial write - Source SPEC comments: -- https://github.com/higress-group/issue-spec/issues/439#issuecomment-5226066355 +- https://github.com/higress-group/issue-spec/issues/323#issuecomment-5035087599 ### Requirement: Runner derives qoder-specific acpx configuration for job dispatch diff --git a/issue-spec/specs/runner-shared-runtime-home/spec.md b/issue-spec/specs/runner-shared-runtime-home/spec.md deleted file mode 100644 index 0a65b0aa..00000000 --- a/issue-spec/specs/runner-shared-runtime-home/spec.md +++ /dev/null @@ -1,103 +0,0 @@ -# runner-shared-runtime-home - -## Purpose - -Define the long-lived behavior contract for this capability. - -Proposal Issues: -- https://github.com/higress-group/issue-spec/issues/439 - -## Requirements - -### Requirement: Runner sessions share one runner-scoped isolated runtime HOME - -The runner MUST give every public session dispatched for one runner scope the same persistent isolated runtime HOME, where the scope identity is the hostname, backend profile realm, canonical repository identity, and runner identity. The runtime HOME MUST never be the operator's real HOME, MUST be created runner-owned with private (0700) permissions strictly confined below the resolved runner workspace root, and MUST fail closed when the resolved path escapes the root, crosses a symlink, or collides with another root entry. Repository content MUST NOT be able to select or redirect the runtime HOME path. Different repositories, runner identities, or backend profile realms MUST NOT share a writable runtime HOME. Session routing MUST continue to use only the exact stored public session ID, ACPX record ID, ACP session ID, and repository binding from durable runner state; the runner MUST NOT infer the most recent or active conversation from runtime HOME contents. - -#### Scenario: concurrent sessions share one runtime HOME - -- **WHEN** two public sessions of the same runner scope dispatch jobs, concurrently or sequentially -- **THEN** both jobs SHALL receive the same runner-scoped runtime HOME while each dispatch keeps its own exact public session ID, ACPX record ID, and managed workspace, and alternating `/resume` operations before and after a runner restart SHALL reconnect each session to its original agent identity - -#### Scenario: different scope cannot reuse the runtime HOME - -- **WHEN** a job is dispatched for a different repository, runner identity, or backend profile realm -- **THEN** the runner SHALL derive a distinct runtime HOME for that scope and SHALL NOT bind another scope's runtime HOME into the sandbox - -#### Scenario: unsafe path resolution fails closed - -- **WHEN** the derived runtime HOME path escapes the workspace root, resolves through a symlink, or overlaps a protected root entry -- **THEN** dispatch SHALL fail with an actionable diagnostic instead of creating or using the unsafe path - -#### Scenario: operator HOME stays untouched - -- **WHEN** any runner job executes in the default sandbox -- **THEN** the operator's real HOME SHALL remain unavailable for general writes and SHALL never be selected as the shared runtime HOME - -Source SPEC comments: -- https://github.com/higress-group/issue-spec/issues/439#issuecomment-5226066432 - -### Requirement: Each dispatched job receives unique disposable scratch directories - -Every dispatched runner job MUST receive its own private scratch directories for `TMPDIR`, `GOTMPDIR`, `XDG_DATA_HOME`, and `XDG_STATE_HOME`, unique per runner job ID and never shared between jobs. Job scratch MUST be removed after the job reaches a terminal state, and scratch left behind by a crashed runner MUST be recovered by a conservative, idempotent reconciliation that never deletes scratch belonging to an active job. Job-private credential material MUST follow the job scratch lifecycle and MUST NOT persist in the shared runtime HOME. - -#### Scenario: scratch is unique and writable - -- **WHEN** two jobs run concurrently in the same runner scope -- **THEN** each job SHALL observe its own sandbox-writable `TMPDIR`, `GOTMPDIR`, `XDG_DATA_HOME`, and `XDG_STATE_HOME` paths that the other job does not receive - -#### Scenario: scratch is removed at terminal completion - -- **WHEN** a job completes, fails, or is cancelled -- **THEN** the runner SHALL remove that job's scratch directories without touching the shared runtime HOME or another job's scratch - -#### Scenario: crash recovery is conservative and idempotent - -- **WHEN** the runner restarts after a crash that left job scratch directories behind -- **THEN** reconciliation SHALL remove only scratch whose owning job is terminal or unknown to durable state, SHALL keep scratch of active jobs, and repeating the pass SHALL be a no-op - -Source SPEC comments: -- https://github.com/higress-group/issue-spec/issues/439#issuecomment-5226066531 - -### Requirement: Shared runtime HOME storage is owner-locked, accounted, and cache-only evictable - -The runner-scoped runtime HOME and per-job scratch MUST be managed physical storage under the same storage owner lock and reconciliation engine that owns session runtimes and PROCESS pools, and reconciliation of these resources MUST stay idempotent across crashes between filesystem and metadata transitions. Storage diagnostics MUST distinguish protected identity/config bytes, rebuildable cache bytes, job scratch bytes, and unknown bytes for the runtime HOME. Storage pressure and cleanup MAY evict rebuildable cache directories and stale job scratch, but MUST NOT remove authentication or configuration state, ACPX index and session mappings, live agent state, active workspaces, or anything required to `/resume` an existing session. Known-expired resources MUST NOT receive a fresh orphan grace period, and an older runner binary reading the newer storage metadata MUST NOT corrupt or silently delete the runtime HOME or its records. - -#### Scenario: storage report classifies runtime HOME bytes - -- **WHEN** an operator runs the storage reconciliation report -- **THEN** the report SHALL show the runtime HOME's protected identity/config bytes, rebuildable cache bytes, job scratch bytes, and unknown bytes as distinct categories without exposing file contents or credentials - -#### Scenario: cache cleanup preserves resume - -- **WHEN** low disk space or an explicit cleanup evicts rebuildable caches from the runtime HOME -- **THEN** eviction SHALL be limited to cache-eligible and stale scratch paths and every existing public session SHALL remain resumable afterwards - -#### Scenario: old binary rollback is safe - -- **WHEN** an older runner binary that predates the shared runtime HOME runs against the upgraded root -- **THEN** the older binary SHALL NOT delete or corrupt the shared runtime HOME, its metadata records, or the storage sidecar's ownership evidence - -Source SPEC comments: -- https://github.com/higress-group/issue-spec/issues/439#issuecomment-5226066600 - -### Requirement: Legacy per-session runtime homes migrate with resume validation - -Upgrading a runner root that contains legacy per-session runtime HOME directories MUST preserve exact `/resume` compatibility for existing sessions. Migration MUST run only while holding exclusive storage owner ownership with the old runner stopped, MUST back up control-plane state and the storage sidecar first, MUST create the runner-scoped runtime HOME with private permissions, MUST import verified ACPX and agent session state while keeping every existing ACPX record ID and agent session identity unchanged, and MUST NOT copy rebuildable cache directories from every legacy session. Legacy runtime homes MUST be preserved until a real resume through the new runtime HOME succeeds, and only then MAY the redundant legacy data be retired through the existing storage reconciler without granting known-expired resources a fresh orphan grace period. When legacy session homes contain conflicting global agent configuration or indexes, migration MUST stop with actionable diagnostics instead of silently choosing one. Migration, retry, and rollback MUST be idempotent, and rolling back to the previous binary MUST NOT corrupt or lose the sidecar or the new runtime HOME metadata. - -#### Scenario: migrated sessions resume with original identity - -- **WHEN** an operator migrates a root with existing per-session runtime homes and then resumes a migrated session -- **THEN** the session SHALL reconnect to its original ACPX record ID and agent session through the shared runtime HOME, and legacy homes SHALL be retired only after that validation succeeds - -#### Scenario: conflicting legacy configuration fails closed - -- **WHEN** two legacy session homes carry conflicting global agent configuration or index files -- **THEN** migration SHALL stop with a diagnostic naming the conflicting sessions and paths and SHALL NOT pick one silently - -#### Scenario: migration retries are idempotent - -- **WHEN** migration is interrupted and re-run, or re-run after it already completed -- **THEN** the repeated run SHALL converge to the same completed state without duplicating imports, corrupting state, or restarting orphan grace for known-expired resources - -Source SPEC comments: -- https://github.com/higress-group/issue-spec/issues/439#issuecomment-5226066701 From 2ab6cfb2bb36e10d979225dbf0ca27cc24243692 Mon Sep 17 00:00:00 2001 From: johnlanni Date: Sat, 8 Aug 2026 21:34:05 +0800 Subject: [PATCH 04/17] docs(specs): materialize corrected durable specs for shared runtime HOME Re-materializes after the clean-cutover correction: SPEC-439005 drops old-binary rollback in favor of the single storage sidecar authority, SPEC-439006 (legacy migration) is superseded, and SPEC-439007 adds the clean-cutover requirement. Signed-off-by: johnlanni --- .../acpx-comment-triggered-workflow/spec.md | 4 +- .../specs/runner-coordinator-agents/spec.md | 9 +- .../specs/runner-shared-runtime-home/spec.md | 103 ++++++++++++++++++ 3 files changed, 112 insertions(+), 4 deletions(-) create mode 100644 issue-spec/specs/runner-shared-runtime-home/spec.md diff --git a/issue-spec/specs/acpx-comment-triggered-workflow/spec.md b/issue-spec/specs/acpx-comment-triggered-workflow/spec.md index 0b0a2cdf..da230d4a 100644 --- a/issue-spec/specs/acpx-comment-triggered-workflow/spec.md +++ b/issue-spec/specs/acpx-comment-triggered-workflow/spec.md @@ -124,7 +124,7 @@ The runner MUST invoke acpx as an external coordinator backend using argv arrays #### Scenario: default sandbox - **WHEN** a runner job dispatches on Linux without `--unsafe-no-sandbox` -- **THEN** acpx SHALL run through bubblewrap with workspace filesystem isolation, temporary `HOME`, `GH_CONFIG_DIR`, `XDG_CONFIG_HOME`, and `CODEX_HOME` when needed, inherited proxy settings, broad token environment variables scrubbed, required system paths read-only, and the managed workspace mounted for writes. +- **THEN** acpx SHALL run through bubblewrap with workspace filesystem isolation, the persistent runner-scoped isolated runtime `HOME` shared by that runner scope's public sessions (never the operator's real HOME) with managed `GH_CONFIG_DIR`, `XDG_CONFIG_HOME`, and `CODEX_HOME` when needed, unique disposable per-job scratch directories for `TMPDIR`, `GOTMPDIR`, `XDG_DATA_HOME`, and `XDG_STATE_HOME`, inherited proxy settings, broad token environment variables scrubbed, required system paths read-only, and the managed workspace mounted for writes. #### Scenario: unsafe mode @@ -152,7 +152,7 @@ The runner MUST invoke acpx as an external coordinator backend using argv arrays - **THEN** the runner SHALL treat acpx as the top-level headless coordinator transport and persist bounded child provenance reported by the coordinator without needing direct access to the native worker runtime. Source SPEC comments: -- https://github.com/higress-group/issue-spec/issues/343#issuecomment-5099614411 +- https://github.com/higress-group/issue-spec/issues/439#issuecomment-5226066261 ### Requirement: durable job state and recovery diff --git a/issue-spec/specs/runner-coordinator-agents/spec.md b/issue-spec/specs/runner-coordinator-agents/spec.md index 6169295b..a690576e 100644 --- a/issue-spec/specs/runner-coordinator-agents/spec.md +++ b/issue-spec/specs/runner-coordinator-agents/spec.md @@ -38,7 +38,7 @@ Source SPEC comments: ### Requirement: Runner mirrors host qoder configuration into the sandbox -The runner MUST mirror the host ~/.qoder/settings.json and the regular files directly inside ~/.qoder/.auth into the sandbox temporary HOME. The .auth mirror MUST be one level only, MUST preserve restrictive source permission bits (defaulting to 0600), MUST remove stale mirrored files, and MUST NOT follow destination symlinks or mirror cache, log, session-state, symlink, directory, or device entries. +The runner MUST mirror the host ~/.qoder/settings.json and the regular files directly inside ~/.qoder/.auth into the sandbox runtime HOME (the persistent runner-scoped isolated HOME exposed to the sandbox as $HOME, referred to below as $TempHome). The .auth mirror MUST be one level only, MUST preserve restrictive source permission bits (defaulting to 0600), MUST remove stale mirrored files, and MUST NOT follow destination symlinks or mirror cache, log, session-state, symlink, directory, or device entries. Because the runtime HOME is shared by concurrent jobs of one runner scope, each mirrored file refresh MUST be atomic and idempotent so a concurrent job never observes a partially written configuration or credential file. #### Scenario: settings.json is mirrored into sandbox TempHome @@ -65,8 +65,13 @@ The runner MUST mirror the host ~/.qoder/settings.json and the regular files dir - **WHEN** the host qoder config contains symlinks, subdirectories, devices, cache, logs, or session state - **THEN** the mirror MUST skip those entries, MUST NOT recurse, and MUST NOT expose them inside the sandbox +#### Scenario: concurrent dispatches refresh the shared mirror safely + +- **WHEN** two jobs of the same runner scope dispatch concurrently and both refresh the qoder mirror in the shared runtime HOME +- **THEN** each mirrored file MUST be replaced atomically so a running agent observes either the previous or the new complete file, never a partial write + Source SPEC comments: -- https://github.com/higress-group/issue-spec/issues/323#issuecomment-5035087599 +- https://github.com/higress-group/issue-spec/issues/439#issuecomment-5226066355 ### Requirement: Runner derives qoder-specific acpx configuration for job dispatch diff --git a/issue-spec/specs/runner-shared-runtime-home/spec.md b/issue-spec/specs/runner-shared-runtime-home/spec.md new file mode 100644 index 00000000..df970295 --- /dev/null +++ b/issue-spec/specs/runner-shared-runtime-home/spec.md @@ -0,0 +1,103 @@ +# runner-shared-runtime-home + +## Purpose + +Define the long-lived behavior contract for this capability. + +Proposal Issues: +- https://github.com/higress-group/issue-spec/issues/439 + +## Requirements + +### Requirement: Runner sessions share one runner-scoped isolated runtime HOME + +The runner MUST give every public session dispatched for one runner scope the same persistent isolated runtime HOME, where the scope identity is the hostname, backend profile realm, canonical repository identity, and runner identity. The runtime HOME MUST never be the operator's real HOME, MUST be created runner-owned with private (0700) permissions strictly confined below the resolved runner workspace root, and MUST fail closed when the resolved path escapes the root, crosses a symlink, or collides with another root entry. Repository content MUST NOT be able to select or redirect the runtime HOME path. Different repositories, runner identities, or backend profile realms MUST NOT share a writable runtime HOME. Session routing MUST continue to use only the exact stored public session ID, ACPX record ID, ACP session ID, and repository binding from durable runner state; the runner MUST NOT infer the most recent or active conversation from runtime HOME contents. + +#### Scenario: concurrent sessions share one runtime HOME + +- **WHEN** two public sessions of the same runner scope dispatch jobs, concurrently or sequentially +- **THEN** both jobs SHALL receive the same runner-scoped runtime HOME while each dispatch keeps its own exact public session ID, ACPX record ID, and managed workspace, and alternating `/resume` operations before and after a runner restart SHALL reconnect each session to its original agent identity + +#### Scenario: different scope cannot reuse the runtime HOME + +- **WHEN** a job is dispatched for a different repository, runner identity, or backend profile realm +- **THEN** the runner SHALL derive a distinct runtime HOME for that scope and SHALL NOT bind another scope's runtime HOME into the sandbox + +#### Scenario: unsafe path resolution fails closed + +- **WHEN** the derived runtime HOME path escapes the workspace root, resolves through a symlink, or overlaps a protected root entry +- **THEN** dispatch SHALL fail with an actionable diagnostic instead of creating or using the unsafe path + +#### Scenario: operator HOME stays untouched + +- **WHEN** any runner job executes in the default sandbox +- **THEN** the operator's real HOME SHALL remain unavailable for general writes and SHALL never be selected as the shared runtime HOME + +Source SPEC comments: +- https://github.com/higress-group/issue-spec/issues/439#issuecomment-5226066432 + +### Requirement: Each dispatched job receives unique disposable scratch directories + +Every dispatched runner job MUST receive its own private scratch directories for `TMPDIR`, `GOTMPDIR`, `XDG_DATA_HOME`, and `XDG_STATE_HOME`, unique per runner job ID and never shared between jobs. Job scratch MUST be removed after the job reaches a terminal state, and scratch left behind by a crashed runner MUST be recovered by a conservative, idempotent reconciliation that never deletes scratch belonging to an active job. Job-private credential material MUST follow the job scratch lifecycle and MUST NOT persist in the shared runtime HOME. + +#### Scenario: scratch is unique and writable + +- **WHEN** two jobs run concurrently in the same runner scope +- **THEN** each job SHALL observe its own sandbox-writable `TMPDIR`, `GOTMPDIR`, `XDG_DATA_HOME`, and `XDG_STATE_HOME` paths that the other job does not receive + +#### Scenario: scratch is removed at terminal completion + +- **WHEN** a job completes, fails, or is cancelled +- **THEN** the runner SHALL remove that job's scratch directories without touching the shared runtime HOME or another job's scratch + +#### Scenario: crash recovery is conservative and idempotent + +- **WHEN** the runner restarts after a crash that left job scratch directories behind +- **THEN** reconciliation SHALL remove only scratch whose owning job is terminal or unknown to durable state, SHALL keep scratch of active jobs, and repeating the pass SHALL be a no-op + +Source SPEC comments: +- https://github.com/higress-group/issue-spec/issues/439#issuecomment-5226066531 + +### Requirement: Shared runtime HOME storage is owner-locked, accounted, and cache-only evictable + +The runner-scoped runtime HOME and per-job scratch MUST be managed physical storage tracked as records in the existing runner storage sidecar under the same owner lock and reconciliation engine that owns session runtimes and PROCESS pools; no parallel lifecycle engine or separate metadata authority is introduced. Reconciliation of these resources MUST stay idempotent across crashes between filesystem and metadata transitions. Storage diagnostics MUST distinguish protected identity/config bytes, rebuildable cache bytes, job scratch bytes, and unknown bytes for the runtime HOME. Storage pressure and cleanup MAY evict rebuildable cache directories and stale job scratch, but MUST NOT remove authentication or configuration state, ACPX index and session mappings, live agent state, active workspaces, or anything required to `/resume` an existing session on the current layout. Known-expired resources MUST NOT receive a fresh orphan grace period. + +#### Scenario: storage report classifies runtime HOME bytes + +- **WHEN** an operator runs the storage reconciliation report +- **THEN** the report SHALL show the runtime HOME's protected identity/config bytes, rebuildable cache bytes, job scratch bytes, and unknown bytes as distinct categories without exposing file contents or credentials + +#### Scenario: cache cleanup preserves resume + +- **WHEN** low disk space or an explicit cleanup evicts rebuildable caches from the runtime HOME +- **THEN** eviction SHALL be limited to cache-eligible and stale scratch paths and every existing public session on the current layout SHALL remain resumable afterwards + +#### Scenario: single storage authority + +- **WHEN** the runner registers, accounts, reconciles, or evicts shared runtime HOME or job scratch resources +- **THEN** those operations SHALL use the existing storage sidecar records and owner lock and SHALL NOT create a second lifecycle engine or metadata file with independent deletion authority + +Source SPEC comments: +- https://github.com/higress-group/issue-spec/issues/439#issuecomment-5226066600 + +### Requirement: Shared runtime HOME is a clean cutover for new runner roots + +The runner-scoped isolated runtime HOME MUST be enabled as a clean operational cutover for newly started runner roots and deployments. The runner MUST NOT migrate, import, or resume legacy per-session runtime HOME state, and operator documentation MUST provide an explicit cutover procedure: stop the old runner, preserve or archive the old root if desired, start the new binary against a fresh runner root, verify new-session concurrency and cache reuse, then remove or archive the old root separately. The new binary MUST NOT modify, import, or delete an old runner root; archiving or deleting it is always a separate explicit operator action. Sessions created before the cutover are not resumable afterwards and are drained or archived by the operator. + +#### Scenario: fresh root initializes the shared layout + +- **WHEN** a new binary starts against a fresh runner root and dispatches new sessions +- **THEN** the runner SHALL create the runner-scoped isolated runtime HOME with private permissions and SHALL serve new sessions and `/resume` entirely within the shared layout, including across runner restarts + +#### Scenario: old root is untouched + +- **WHEN** the new binary runs against a fresh root while an old runner root still exists elsewhere on the host +- **THEN** the new binary SHALL NOT read for import, modify, or delete the old root, and removing or archiving it SHALL remain a separate explicit operator action + +#### Scenario: cutover is documented + +- **WHEN** an operator adopts the shared runtime HOME +- **THEN** runner documentation SHALL describe the breaking cutover, the drain-or-archive expectation for pre-cutover sessions, and the step-by-step cutover procedure + +Source SPEC comments: +- https://github.com/higress-group/issue-spec/issues/439#issuecomment-5226324113 From 1f638c404a198abae79c5dee053ddad30805fd48 Mon Sep 17 00:00:00 2001 From: johnlanni Date: Sat, 8 Aug 2026 21:54:03 +0800 Subject: [PATCH 05/17] refactor(runner): track runtime homes in storage sidecar; drop migration machinery Rework the phase-1 storage foundations to the corrected clean-cutover model (maintainer correction on higress-group/issue-spec#439, design higress-group/issue-spec#440): no legacy migration, no resume preservation across upgrade, no migration ledger, no old-binary rollback support. - Delete runtime_store.go (the parallel .storage/runtime.json metadata store) and migrate.go (two-phase legacy import + retirement) with their tests. The existing storage sidecar is the single metadata authority for every physical resource. - storage.go gains runner_home and job_scratch resource kinds recorded as ordinary PhysicalResource entries (scope repo / job repo in the repo segment, empty session segment, scope hash / job ID in the hash segment). - RecordRuntimeHome, RecordJobScratch, and CompleteJobScratch now run locked read-modify-write cycles on the shared sidecar with skip-unchanged upserts; CompleteJobScratch is idempotent and leaves crash consistency to the generic engine sweep. - ReconcileJobScratch and EvictRuntimeCaches iterate sidecar records under the same owner lock, keeping active-job protection, foreign name rejection, dry-run reporting, and second-pass no-op behavior. - AdmitDispatch's pressured pass additionally reclaims stale job scratch and rebuildable home caches (best-effort, bounded diagnostics) once the root carries at least one runner_home record. - RecordSessionProcessPool is the thin pool-only recording variant for the shared layout, where no per-session .sessions/ runtime exists. - New tests cover kind validation, record shape/idempotency, the generic sweep dropping stale scratch records, runner_home records surviving an apply reconcile, and pressured admission reclaiming scratch and caches (gated on shared-layout adoption). Refs higress-group/issue-spec#439, design higress-group/issue-spec#440. Signed-off-by: johnlanni --- internal/commentrunner/storage/migrate.go | 684 ------------------ .../commentrunner/storage/migrate_test.go | 576 --------------- .../storage/runtime_reconcile.go | 107 +-- .../storage/runtime_reconcile_test.go | 109 +-- .../storage/runtime_sidecar_test.go | 290 ++++++++ .../commentrunner/storage/runtime_store.go | 387 ---------- .../storage/runtime_store_test.go | 238 ------ internal/commentrunner/storage/service.go | 348 ++++----- internal/commentrunner/storage/storage.go | 14 +- 9 files changed, 571 insertions(+), 2182 deletions(-) delete mode 100644 internal/commentrunner/storage/migrate.go delete mode 100644 internal/commentrunner/storage/migrate_test.go create mode 100644 internal/commentrunner/storage/runtime_sidecar_test.go delete mode 100644 internal/commentrunner/storage/runtime_store.go delete mode 100644 internal/commentrunner/storage/runtime_store_test.go diff --git a/internal/commentrunner/storage/migrate.go b/internal/commentrunner/storage/migrate.go deleted file mode 100644 index 17699073..00000000 --- a/internal/commentrunner/storage/migrate.go +++ /dev/null @@ -1,684 +0,0 @@ -package storage - -import ( - "bytes" - "context" - "crypto/sha256" - "encoding/hex" - "errors" - "fmt" - "io" - "os" - "path/filepath" - "sort" - "strings" - - "github.com/higress-group/issue-spec/internal/commentrunner/state" -) - -// runtimeBackupName preserves the runtime metadata before the first applied -// migration, alongside the raw state and sidecar first-migration backups. -const runtimeBackupName = "runtime-first.json" - -// migrationDestinationMarker names the existing shared-home file in a -// conflict's Sessions list. It can never collide with a session key, which -// always has the repo#id shape. -const migrationDestinationMarker = "(shared-home)" - -// MigrateHomeOptions controls one legacy-home migration pass. -type MigrateHomeOptions struct { - Scope RuntimeScope - Apply bool -} - -// MigrationConflict identifies one destination relpath produced with -// different content by multiple legacy sessions, or already present at the -// destination with different content. Sessions lists the conflicting session -// keys, plus migrationDestinationMarker when the destination participates. -type MigrationConflict struct { - RelPath string `json:"rel_path"` - Sessions []string `json:"sessions"` -} - -// MigrateHomeReport summarizes one migration or retirement pass. It carries -// paths, counts, and session identities only — never file contents. -type MigrateHomeReport struct { - ScopeHash string `json:"scope_hash"` - LedgerState MigrationState `json:"ledger_state,omitempty"` - ImportedSessions []string `json:"imported_sessions,omitempty"` - SkippedIdentical int `json:"skipped_identical,omitempty"` - CopiedFiles int `json:"copied_files,omitempty"` - Conflicts []MigrationConflict `json:"conflicts,omitempty"` - RetiredLegacy []string `json:"retired_legacy,omitempty"` - Diagnostics []string `json:"diagnostics,omitempty"` -} - -// importSourceSpec maps one legacy runtime subtree into the shared home. -// Only the listed subtrees are ever imported; caches, logs, sockets, and -// per-job mirrored credentials are deliberately excluded. -type importSourceSpec struct { - srcSub string - destSub string - excludeTop map[string]bool - excludeDir map[string]bool -} - -// codexMirroredTopLevel are refreshed from the host on every dispatch, so the -// legacy copies are never imported. -var codexMirroredTopLevel = map[string]bool{ - "auth.json": true, "config.toml": true, "version.json": true, "installation_id": true, -} - -var claudeMirroredTopLevel = map[string]bool{ - "settings.json": true, "settings.local.json": true, ".credentials.json": true, -} - -var qoderMirroredTopLevel = map[string]bool{"settings.json": true} - -var legacyImportSources = []importSourceSpec{ - {srcSub: filepath.Join("home", ".acpx", "sessions"), destSub: filepath.Join("home", ".acpx", "sessions")}, - {srcSub: filepath.Join("home", ".acpx", "queues"), destSub: filepath.Join("home", ".acpx", "queues")}, - {srcSub: "codex", destSub: "codex", excludeTop: codexMirroredTopLevel}, - {srcSub: filepath.Join("home", ".codex"), destSub: filepath.Join("home", ".codex"), excludeTop: codexMirroredTopLevel}, - {srcSub: filepath.Join("home", ".claude"), destSub: filepath.Join("home", ".claude"), excludeTop: claudeMirroredTopLevel}, - {srcSub: filepath.Join("home", ".qoder"), destSub: filepath.Join("home", ".qoder"), excludeTop: qoderMirroredTopLevel, excludeDir: map[string]bool{".auth": true}}, -} - -// legacySessionSource is one selected session's verified legacy runtime root. -type legacySessionSource struct { - key string - root string -} - -// importCandidate is the chosen source file for one destination relpath. -type importCandidate struct { - src string - digest string - exec bool - session string -} - -// migrationPlan is the phase-A result: the exact copies to perform, the -// already-identical destinations, and every detected conflict. -type migrationPlan struct { - copies map[string]importCandidate - skipped int - conflicts []MigrationConflict -} - -// MigrateHome imports legacy per-session agent state into the runner-scoped -// shared runtime home. It runs only under the storage owner, backs up -// control-plane state first, and is two-phase: a full scan across every -// selected session builds the destination plan, and any conflict aborts -// before a single import write. Apply is idempotent; a dry run performs the -// full scan and reports without touching the filesystem or the ledger. -func (s *Service) MigrateHome(ctx context.Context, opts MigrateHomeOptions) (MigrateHomeReport, error) { - if err := opts.Scope.Validate(); err != nil { - return MigrateHomeReport{}, err - } - scopeHash, err := RuntimeScopeHash(opts.Scope) - if err != nil { - return MigrateHomeReport{}, err - } - report := MigrateHomeReport{ScopeHash: scopeHash} - owner, release, err := EnsureOwner(ctx, s.root) - if err != nil { - return report, fmt.Errorf("migration owner: %w", err) - } - defer release() - ctx = WithOwner(ctx, owner) - - if opts.Apply { - if err := s.migrationBackups(); err != nil { - return report, fmt.Errorf("migration backups: %w", err) - } - } - st, err := s.stateLoader(ctx) - if err != nil { - return report, fmt.Errorf("load runner state for migration: %w", err) - } - st.Normalize() - sources := s.selectLegacySessions(st, opts.Scope.Repo, &report) - for _, source := range sources { - report.ImportedSessions = append(report.ImportedSessions, source.key) - } - - sharedRoot := filepath.Join(s.root, RunnerHomesDirName, scopeHash) - if opts.Apply { - if _, err := PrepareRuntimeHome(s.root, opts.Scope); err != nil { - return report, err - } - } else { - // Dry runs validate an existing binding but never create the tree. - if _, statErr := os.Lstat(sharedRoot); statErr == nil { - if err := checkRuntimeScopeBinding(sharedRoot, opts.Scope); err != nil { - return report, err - } - } else if !errors.Is(statErr, os.ErrNotExist) { - return report, fmt.Errorf("inspect runtime home: %w", statErr) - } - } - - plan, err := buildMigrationPlan(sources, sharedRoot) - if err != nil { - return report, err - } - report.Conflicts = plan.conflicts - if len(plan.conflicts) > 0 { - return report, fmt.Errorf("migration has %d conflicting paths; resolve the legacy homes manually before importing", len(plan.conflicts)) - } - report.SkippedIdentical = plan.skipped - if !opts.Apply { - report.CopiedFiles = len(plan.copies) - report.Diagnostics = append(report.Diagnostics, "dry-run: no filesystem or ledger writes performed") - return report, nil - } - if err := applyMigrationPlan(sharedRoot, plan, &report); err != nil { - return report, err - } - // The ledger never regresses: a re-run after validation keeps the advanced - // state while still union-merging the imported session set. - markState := MigrationImported - existing, has, err := s.RuntimeMigrationLedger(ctx, scopeHash) - if err != nil { - return report, err - } - if has && migrationStateRank(existing.State) > migrationStateRank(MigrationImported) { - markState = existing.State - } - if err := s.MarkRuntimeMigration(ctx, MigrationRecord{ - ScopeHash: scopeHash, - State: markState, - ImportedSessions: report.ImportedSessions, - ValidatedSession: existing.ValidatedSession, - }); err != nil { - return report, err - } - ledger, _, err := s.RuntimeMigrationLedger(ctx, scopeHash) - if err != nil { - return report, err - } - report.LedgerState = ledger.State - return report, nil -} - -// RetireLegacyHomes retires the legacy per-session runtime dirs of an -// already validated migration: the matching sidecar session_runtime records -// are marked retired_known (never a fresh orphan grace) and one engine -// reconcile pass removes them through the existing deletion transactions. The -// ledger advances to retired once the pass succeeds; re-runs are idempotent. -func (s *Service) RetireLegacyHomes(ctx context.Context, scope RuntimeScope, apply bool) (MigrateHomeReport, error) { - if err := scope.Validate(); err != nil { - return MigrateHomeReport{}, err - } - scopeHash, err := RuntimeScopeHash(scope) - if err != nil { - return MigrateHomeReport{}, err - } - report := MigrateHomeReport{ScopeHash: scopeHash} - owner, release, err := EnsureOwner(ctx, s.root) - if err != nil { - return report, fmt.Errorf("retire legacy homes owner: %w", err) - } - defer release() - ctx = WithOwner(ctx, owner) - - ledger, ok, err := s.RuntimeMigrationLedger(ctx, scopeHash) - if err != nil { - return report, err - } - if !ok { - return report, fmt.Errorf("migration ledger for scope hash %q not found; run the import first", scopeHash) - } - if ledger.State != MigrationValidated && ledger.State != MigrationRetired { - return report, fmt.Errorf("migration ledger for scope hash %q is %q; a validated resume through the shared home is required before retiring legacy homes", scopeHash, ledger.State) - } - report.LedgerState = ledger.State - report.ImportedSessions = append([]string(nil), ledger.ImportedSessions...) - - st, err := s.stateLoader(ctx) - if err != nil { - return report, fmt.Errorf("load runner state for legacy retirement: %w", err) - } - st.Normalize() - expectedRoots := s.legacyRootsBySession(st, ledger.ImportedSessions, &report) - - // Only sidecar records whose ownership names an imported session and whose - // path matches that session's exact legacy runtime root may retire. For - // sessions already pruned from state the sidecar's proven ownership fields - // are the path evidence. - sidecar := s.store.State() - type retireTarget struct { - id string - sessionKey string - path string - } - var targets []retireTarget - for id, resource := range sidecar.Resources { - if resource.Kind != ResourceKindSessionRuntime || !resource.Owned() { - continue - } - key := state.PublicSessionKey(resource.Repo, resource.PublicSessionID) - if !containsString(ledger.ImportedSessions, key) { - continue - } - if expected, ok := expectedRoots[key]; ok && filepath.Clean(resource.Path) != expected { - report.Diagnostics = append(report.Diagnostics, safeDiagnostic("sidecar record "+id+" path does not match the session runtime root; skipped")) - continue - } - targets = append(targets, retireTarget{id: id, sessionKey: key, path: resource.Path}) - } - sort.Slice(targets, func(i, j int) bool { return targets[i].id < targets[j].id }) - - if !apply { - for _, target := range targets { - report.Diagnostics = append(report.Diagnostics, "would retire legacy runtime for session "+target.sessionKey) - } - report.Diagnostics = append(report.Diagnostics, "dry-run: no filesystem or ledger writes performed") - return report, nil - } - if err := s.store.Update(func(st *StorageState) error { - for _, target := range targets { - current, ok := st.Resources[target.id] - if !ok { - continue - } - switch current.CleanupState { - case CleanupDeleting, CleanupRemoved: - continue - } - current.CleanupState = CleanupRetiredKnown - current.CleanupAttemptID = "" - current.LastCleanupError = "" - st.Resources[target.id] = current - } - return nil - }); err != nil { - return report, fmt.Errorf("mark legacy runtimes retired: %w", err) - } - if _, err := s.ReconcileStorage(ctx, true, false); err != nil { - return report, fmt.Errorf("retire reconcile pass: %w", err) - } - if err := s.MarkRuntimeMigration(ctx, MigrationRecord{ - ScopeHash: scopeHash, - State: MigrationRetired, - ImportedSessions: ledger.ImportedSessions, - ValidatedSession: ledger.ValidatedSession, - }); err != nil { - return report, err - } - report.LedgerState = MigrationRetired - for _, target := range targets { - if _, err := os.Lstat(target.path); errors.Is(err, os.ErrNotExist) { - report.RetiredLegacy = append(report.RetiredLegacy, target.sessionKey) - } else { - report.Diagnostics = append(report.Diagnostics, safeDiagnostic("legacy runtime for session "+target.sessionKey+" preserved by reconciliation")) - } - } - sort.Strings(report.RetiredLegacy) - return report, nil -} - -// migrationBackups performs the first-migration preservation set: the raw -// pre-Normalize runner state (when configured), the sidecar, and the runtime -// metadata. All three are atomic, idempotent, and never overwritten. -func (s *Service) migrationBackups() error { - if _, err := EnsureRawStateBackup(s.root, s.rawStatePath); err != nil { - return err - } - if err := ensureMetadataBackup(s.root, s.store.Path(), sidecarBackupPrefix+".json"); err != nil { - return err - } - runtimeStore, err := s.RuntimeStore() - if err != nil { - return err - } - return ensureMetadataBackup(s.root, runtimeStore.Path(), runtimeBackupName) -} - -// ensureMetadataBackup atomically preserves the first copy of one metadata -// file under `.storage/backups/`, never overwriting an earlier copy. -func ensureMetadataBackup(workspaceRoot, sourcePath, backupName string) error { - data, err := os.ReadFile(sourcePath) - if errors.Is(err, os.ErrNotExist) { - return nil - } - if err != nil { - return fmt.Errorf("read %s for migration backup: %w", backupName, err) - } - if len(bytes.TrimSpace(data)) == 0 { - return nil - } - canonical, err := Canonicalize(workspaceRoot) - if err != nil { - return err - } - backupDir := filepath.Join(canonical, StorageDirName, backupDirName) - if err := os.MkdirAll(backupDir, 0o700); err != nil { - return err - } - target := filepath.Join(backupDir, backupName) - if _, err := os.Lstat(target); err == nil { - return nil - } - return state.WriteAtomic(target, data) -} - -// selectLegacySessions picks the repo's retained public sessions whose legacy -// runtime root exists below this root with a home/ subdir. Sessions without -// workspace metadata or with roots outside this root are skipped with a -// diagnostic: foreign paths are never imported. -func (s *Service) selectLegacySessions(st state.RunnerState, repo string, report *MigrateHomeReport) []legacySessionSource { - keys := make([]string, 0, len(st.PublicSessions)) - for key := range st.PublicSessions { - keys = append(keys, key) - } - sort.Strings(keys) - var sources []legacySessionSource - for _, key := range keys { - session := st.PublicSessions[key] - if session.Repo != repo { - continue - } - wsPath := strings.TrimSpace(session.Workspace.Path) - if wsPath == "" { - report.Diagnostics = append(report.Diagnostics, "session "+key+" lacks workspace metadata; skipped") - continue - } - hash, err := SessionRuntimeHash(session.Repo, session.PublicSessionID, wsPath) - if err != nil { - report.Diagnostics = append(report.Diagnostics, safeDiagnostic("session "+key+" runtime hash failed: "+err.Error())) - continue - } - legacyRoot, err := SessionRuntimeRoot(wsPath, session.Repo, session.PublicSessionID) - if err != nil { - report.Diagnostics = append(report.Diagnostics, safeDiagnostic("session "+key+" runtime root failed: "+err.Error())) - continue - } - if expected := filepath.Join(s.root, SessionsDirName, hash); legacyRoot != expected { - report.Diagnostics = append(report.Diagnostics, safeDiagnostic("session "+key+" legacy runtime root is outside this workspace root; skipped")) - continue - } - if info, statErr := os.Lstat(legacyRoot); statErr != nil || info.Mode()&os.ModeSymlink != 0 || !info.IsDir() { - continue - } - if info, statErr := os.Lstat(filepath.Join(legacyRoot, "home")); statErr != nil || info.Mode()&os.ModeSymlink != 0 || !info.IsDir() { - continue - } - sources = append(sources, legacySessionSource{key: key, root: legacyRoot}) - } - return sources -} - -// legacyRootsBySession recomputes the exact legacy runtime root for imported -// sessions whose workspace metadata survives in state. -func (s *Service) legacyRootsBySession(st state.RunnerState, sessionKeys []string, report *MigrateHomeReport) map[string]string { - roots := map[string]string{} - for _, key := range sessionKeys { - session, ok := st.PublicSessions[key] - if !ok { - continue - } - wsPath := strings.TrimSpace(session.Workspace.Path) - if wsPath == "" { - continue - } - hash, err := SessionRuntimeHash(session.Repo, session.PublicSessionID, wsPath) - if err != nil { - report.Diagnostics = append(report.Diagnostics, safeDiagnostic("session "+key+" runtime hash failed: "+err.Error())) - continue - } - roots[key] = filepath.Join(s.root, SessionsDirName, hash) - } - return roots -} - -// buildMigrationPlan scans every selected session's importable subtrees and -// resolves each destination relpath to one content digest. A relpath produced -// with different digests by two sessions, or already present at the -// destination with a different digest, is a conflict. -func buildMigrationPlan(sources []legacySessionSource, sharedRoot string) (migrationPlan, error) { - plan := migrationPlan{copies: map[string]importCandidate{}} - type occurrence struct { - digest string - session string - } - occurrences := map[string][]occurrence{} - chosen := map[string]importCandidate{} - for _, source := range sources { - for _, spec := range legacyImportSources { - srcBase := filepath.Join(source.root, spec.srcSub) - info, err := os.Lstat(srcBase) - if errors.Is(err, os.ErrNotExist) { - continue - } - if err != nil { - return plan, fmt.Errorf("inspect legacy import source: %w", err) - } - if info.Mode()&os.ModeSymlink != 0 || !info.IsDir() { - continue - } - walkErr := filepath.WalkDir(srcBase, func(path string, entry os.DirEntry, err error) error { - if err != nil { - return err - } - rel, err := filepath.Rel(srcBase, path) - if err != nil { - return err - } - if rel == "." { - return nil - } - slashRel := filepath.ToSlash(rel) - top, _, _ := strings.Cut(slashRel, "/") - depthOne := !strings.Contains(slashRel, "/") - if entry.IsDir() { - if spec.excludeDir[top] || (depthOne && spec.excludeTop[top]) { - return filepath.SkipDir - } - return nil - } - // Symlinks, devices, sockets: never imported. - if !entry.Type().IsRegular() { - return nil - } - if depthOne && spec.excludeTop[top] { - return nil - } - digest, _, err := digestFileBytes(path) - if err != nil { - return err - } - fileInfo, err := entry.Info() - if err != nil { - return err - } - destRel := filepath.Join(spec.destSub, rel) - occurrences[destRel] = append(occurrences[destRel], occurrence{digest: digest, session: source.key}) - if _, ok := chosen[destRel]; !ok { - chosen[destRel] = importCandidate{ - src: path, - digest: digest, - exec: fileInfo.Mode().Perm()&0o111 != 0, - session: source.key, - } - } - return nil - }) - if walkErr != nil { - return plan, fmt.Errorf("scan legacy session %s: %w", source.key, walkErr) - } - } - } - - rels := make([]string, 0, len(occurrences)) - for rel := range occurrences { - rels = append(rels, rel) - } - sort.Strings(rels) - for _, rel := range rels { - occs := occurrences[rel] - digests := map[string]bool{} - sessions := map[string]bool{} - for _, occ := range occs { - digests[occ.digest] = true - sessions[occ.session] = true - } - candidate := chosen[rel] - conflictSessions := func(withDestination bool) []string { - set := map[string]bool{} - for session := range sessions { - set[session] = true - } - if withDestination { - set[migrationDestinationMarker] = true - } - list := make([]string, 0, len(set)) - for session := range set { - list = append(list, session) - } - sort.Strings(list) - return list - } - dest := filepath.Join(sharedRoot, rel) - destInfo, statErr := os.Lstat(dest) - switch { - case errors.Is(statErr, os.ErrNotExist): - // No destination yet. - case statErr != nil: - return plan, fmt.Errorf("inspect migration destination: %w", statErr) - case destInfo.Mode().IsRegular() && destInfo.Mode()&os.ModeSymlink == 0: - destDigest, _, err := digestFileBytes(dest) - if err != nil { - return plan, err - } - if len(digests) == 1 && digests[destDigest] { - // Already imported identically: re-runs converge. - plan.skipped++ - continue - } - plan.conflicts = append(plan.conflicts, MigrationConflict{RelPath: filepath.ToSlash(rel), Sessions: conflictSessions(true)}) - continue - default: - // A symlink, directory, or special file at the destination is - // never overwritten. - plan.conflicts = append(plan.conflicts, MigrationConflict{RelPath: filepath.ToSlash(rel), Sessions: conflictSessions(true)}) - continue - } - if len(digests) > 1 { - plan.conflicts = append(plan.conflicts, MigrationConflict{RelPath: filepath.ToSlash(rel), Sessions: conflictSessions(false)}) - continue - } - plan.copies[rel] = candidate - } - return plan, nil -} - -// applyMigrationPlan performs phase B: atomic temp-and-rename copies into the -// shared home, 0700 directories and 0600 files (0700 when the source is -// executable). Destinations are re-validated against the plan so a concurrent -// change fails closed instead of being overwritten. -func applyMigrationPlan(sharedRoot string, plan migrationPlan, report *MigrateHomeReport) error { - rels := make([]string, 0, len(plan.copies)) - for rel := range plan.copies { - rels = append(rels, rel) - } - sort.Strings(rels) - for _, rel := range rels { - candidate := plan.copies[rel] - dest := filepath.Join(sharedRoot, rel) - info, err := os.Lstat(dest) - switch { - case errors.Is(err, os.ErrNotExist): - case err != nil: - return fmt.Errorf("inspect migration destination: %w", err) - default: - if info.Mode().IsRegular() && info.Mode()&os.ModeSymlink == 0 { - if digest, _, digestErr := digestFileBytes(dest); digestErr == nil && digest == candidate.digest { - report.SkippedIdentical++ - continue - } - } - return fmt.Errorf("migration destination %q changed during import; refusing to overwrite", dest) - } - if err := os.MkdirAll(filepath.Dir(dest), 0o700); err != nil { - return fmt.Errorf("prepare migration destination directory: %w", err) - } - if err := copyFileAtomic(dest, candidate.src, candidate.exec); err != nil { - return fmt.Errorf("copy migration file: %w", err) - } - report.CopiedFiles++ - } - return nil -} - -// copyFileAtomic writes src to dest through a temp file in the destination -// directory plus rename, so a concurrent reader never observes a partial -// import. -func copyFileAtomic(dest, src string, exec bool) error { - in, err := os.Open(src) - if err != nil { - return err - } - defer in.Close() - dir := filepath.Dir(dest) - tmp, err := os.CreateTemp(dir, ".migrate-tmp-*") - if err != nil { - return err - } - tmpName := tmp.Name() - cleanup := true - defer func() { - if cleanup { - os.Remove(tmpName) - } - }() - if _, err := io.Copy(tmp, in); err != nil { - _ = tmp.Close() - return err - } - perm := os.FileMode(0o600) - if exec { - perm = 0o700 - } - if err := tmp.Chmod(perm); err != nil { - _ = tmp.Close() - return err - } - if err := tmp.Sync(); err != nil { - _ = tmp.Close() - return err - } - if err := tmp.Close(); err != nil { - return err - } - if err := os.Rename(tmpName, dest); err != nil { - return err - } - cleanup = false - return nil -} - -// digestFileBytes hashes one regular file without reading it into memory. -func digestFileBytes(path string) (string, int64, error) { - file, err := os.Open(path) - if err != nil { - return "", 0, err - } - defer file.Close() - hash := sha256.New() - size, err := io.Copy(hash, file) - if err != nil { - return "", 0, err - } - return hex.EncodeToString(hash.Sum(nil)), size, nil -} - -func containsString(values []string, needle string) bool { - for _, value := range values { - if value == needle { - return true - } - } - return false -} diff --git a/internal/commentrunner/storage/migrate_test.go b/internal/commentrunner/storage/migrate_test.go deleted file mode 100644 index a7f78357..00000000 --- a/internal/commentrunner/storage/migrate_test.go +++ /dev/null @@ -1,576 +0,0 @@ -package storage - -import ( - "context" - "os" - "path/filepath" - "strings" - "testing" - "time" - - "github.com/higress-group/issue-spec/internal/commentrunner/state" -) - -type migrateFixture struct { - t *testing.T - root string - svc *Service - st state.RunnerState - scope RuntimeScope - scopeHash string - rawPath string -} - -func writeContent(t *testing.T, path, content string, perm os.FileMode) { - t.Helper() - if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil { - t.Fatalf("mkdir: %v", err) - } - if err := os.WriteFile(path, []byte(content), perm); err != nil { - t.Fatalf("write %q: %v", path, err) - } -} - -func newMigrateFixture(t *testing.T) *migrateFixture { - t.Helper() - root := testRoot(t) - scope := testScope() - hash, err := RuntimeScopeHash(scope) - if err != nil { - t.Fatalf("scope hash: %v", err) - } - f := &migrateFixture{ - t: t, - root: root, - st: state.NewState(), - scope: scope, - scopeHash: hash, - rawPath: filepath.Join(root, "runner-state.json"), - } - writeContent(t, f.rawPath, `{"schema_version":6}`, 0o600) - svc, err := NewService(ServiceConfig{ - WorkspaceRoot: root, - StateLoader: func(context.Context) (state.RunnerState, error) { return f.st, nil }, - RawStatePath: f.rawPath, - OrphanGrace: DefaultOrphanGrace, - }) - if err != nil { - t.Fatalf("NewService: %v", err) - } - t.Cleanup(func() { _ = svc.Close() }) - f.svc = svc - return f -} - -func (f *migrateFixture) addSession(sid string) string { - f.t.Helper() - wsPath := filepath.Join(f.root, "ws-"+sid) - if err := os.MkdirAll(wsPath, 0o700); err != nil { - f.t.Fatalf("mkdir workspace: %v", err) - } - session := terminalSession(f.scope.Repo, sid, "ws-"+sid, wsPath, time.Now().Add(-time.Hour)) - if err := f.st.UpsertPublicSession(session); err != nil { - f.t.Fatalf("upsert session: %v", err) - } - return wsPath -} - -func (f *migrateFixture) legacyRoot(sid string) string { - f.t.Helper() - session := f.st.PublicSessions[state.PublicSessionKey(f.scope.Repo, sid)] - root, err := SessionRuntimeRoot(session.Workspace.Path, f.scope.Repo, sid) - if err != nil { - f.t.Fatalf("legacy root: %v", err) - } - return root -} - -func (f *migrateFixture) sharedHome() string { - return filepath.Join(f.root, RunnerHomesDirName, f.scopeHash) -} - -func (f *migrateFixture) sessionKeys(sids ...string) []string { - keys := make([]string, 0, len(sids)) - for _, sid := range sids { - keys = append(keys, state.PublicSessionKey(f.scope.Repo, sid)) - } - return keys -} - -// seedLegacyHome writes the standard import fixture into one session's legacy -// runtime root: importable agent state, per-job mirrored files that must be -// skipped, and caches that must never be imported. -func seedLegacyHome(t *testing.T, root, indexContent string) { - t.Helper() - writeContent(t, filepath.Join(root, "home", ".acpx", "sessions", "index.json"), indexContent, 0o600) - writeContent(t, filepath.Join(root, "home", ".acpx", "queues", "q1", "item.json"), "queue-item", 0o600) - writeContent(t, filepath.Join(root, "codex", "sessions", "s1.json"), "codex-session", 0o600) - writeContent(t, filepath.Join(root, "codex", "tools", "run.sh"), "#!/bin/sh\n", 0o755) - writeContent(t, filepath.Join(root, "home", ".claude", "projects", "p.json"), "claude-project", 0o600) - writeContent(t, filepath.Join(root, "home", ".qoder", "mcp.json"), "qoder-mcp", 0o600) - // Mirrored per job from the host: never imported. - writeContent(t, filepath.Join(root, "codex", "auth.json"), "codex-auth", 0o600) - writeContent(t, filepath.Join(root, "codex", "config.toml"), "codex-config", 0o600) - writeContent(t, filepath.Join(root, "home", ".codex", "version.json"), "codex-version", 0o600) - writeContent(t, filepath.Join(root, "home", ".codex", "installation_id"), "codex-install", 0o600) - writeContent(t, filepath.Join(root, "home", ".claude", "settings.json"), "claude-settings", 0o600) - writeContent(t, filepath.Join(root, "home", ".claude", "settings.local.json"), "claude-local", 0o600) - writeContent(t, filepath.Join(root, "home", ".claude", ".credentials.json"), "claude-creds", 0o600) - writeContent(t, filepath.Join(root, "home", ".qoder", "settings.json"), "qoder-settings", 0o600) - writeContent(t, filepath.Join(root, "home", ".qoder", ".auth", "token"), "qoder-token", 0o600) - // Rebuildable or runtime-local state: never imported. - writeContent(t, filepath.Join(root, "home", ".cache", "big"), "cache", 0o600) - writeContent(t, filepath.Join(root, "home", "go", "pkg", "mod", "m.zip"), "mod", 0o600) - writeContent(t, filepath.Join(root, "home", ".npm", "registry.tgz"), "npm", 0o600) - writeContent(t, filepath.Join(root, "gh", "hosts.yml"), "gh", 0o600) - writeContent(t, filepath.Join(root, "acpx-runtime", "sock"), "runtime", 0o600) -} - -func readContent(t *testing.T, path string) string { - t.Helper() - data, err := os.ReadFile(path) - if err != nil { - t.Fatalf("read %q: %v", path, err) - } - return string(data) -} - -func TestMigrateHomeHappyImport(t *testing.T) { - f := newMigrateFixture(t) - f.addSession("ps-1") - f.addSession("ps-2") - seedLegacyHome(t, f.legacyRoot("ps-1"), "shared-index") - writeContent(t, filepath.Join(f.legacyRoot("ps-1"), "home", ".acpx", "sessions", "sess-a.json"), "session-a", 0o600) - seedLegacyHome(t, f.legacyRoot("ps-2"), "shared-index") - writeContent(t, filepath.Join(f.legacyRoot("ps-2"), "home", ".acpx", "sessions", "sess-b.json"), "session-b", 0o600) - // ps-2 carries no claude/qoder/codex extras; identical index dedupes. - - report, err := f.svc.MigrateHome(context.Background(), MigrateHomeOptions{Scope: f.scope, Apply: true}) - if err != nil { - t.Fatalf("MigrateHome: %v", err) - } - if report.ScopeHash != f.scopeHash { - t.Fatalf("scope hash = %q, want %q", report.ScopeHash, f.scopeHash) - } - if report.LedgerState != MigrationImported { - t.Fatalf("ledger state = %q, want %q", report.LedgerState, MigrationImported) - } - if len(report.Conflicts) != 0 { - t.Fatalf("unexpected conflicts: %+v", report.Conflicts) - } - for _, key := range f.sessionKeys("ps-1", "ps-2") { - if !contains(report.ImportedSessions, key) { - t.Fatalf("imported sessions %v missing %q", report.ImportedSessions, key) - } - } - // Unique destinations: index deduped, sess-a/sess-b merged, queue item, - // codex session + executable tool, claude project, qoder mcp. - if report.CopiedFiles != 8 { - t.Fatalf("copied = %d, want 8", report.CopiedFiles) - } - home := f.sharedHome() - wantFiles := map[string]string{ - "home/.acpx/sessions/index.json": "shared-index", - "home/.acpx/sessions/sess-a.json": "session-a", - "home/.acpx/sessions/sess-b.json": "session-b", - "home/.acpx/queues/q1/item.json": "queue-item", - "codex/sessions/s1.json": "codex-session", - "home/.claude/projects/p.json": "claude-project", - "home/.qoder/mcp.json": "qoder-mcp", - } - for rel, content := range wantFiles { - got := readContent(t, filepath.Join(home, filepath.FromSlash(rel))) - if got != content { - t.Fatalf("%s = %q, want %q", rel, got, content) - } - } - // Private perms: 0600 files, 0700 for executables. - info, err := os.Lstat(filepath.Join(home, "home", ".acpx", "sessions", "sess-a.json")) - if err != nil || info.Mode().Perm() != 0o600 { - t.Fatalf("imported file perm = %v, err=%v; want 0600", info.Mode(), err) - } - tool := filepath.Join(home, "codex", "tools", "run.sh") - info, err = os.Lstat(tool) - if err != nil || info.Mode().Perm() != 0o700 { - t.Fatalf("imported executable perm = %v, err=%v; want 0700", info.Mode(), err) - } - // Mirrored and cache content must not be imported. - absent := []string{ - "codex/auth.json", "codex/config.toml", - "home/.codex/version.json", "home/.codex/installation_id", - "home/.claude/settings.json", "home/.claude/settings.local.json", "home/.claude/.credentials.json", - "home/.qoder/settings.json", "home/.qoder/.auth", - "home/.cache", "home/go", "home/.npm", - "gh/hosts.yml", "acpx-runtime/sock", - } - for _, rel := range absent { - if _, err := os.Lstat(filepath.Join(home, filepath.FromSlash(rel))); !os.IsNotExist(err) { - t.Fatalf("%s must not be imported, err=%v", rel, err) - } - } - // Scope binding written; ledger recorded; raw state backup taken. - if _, err := os.Lstat(filepath.Join(home, "scope.json")); err != nil { - t.Fatalf("scope.json missing: %v", err) - } - ledger, ok, err := f.svc.RuntimeMigrationLedger(context.Background(), f.scopeHash) - if err != nil || !ok { - t.Fatalf("ledger missing: %v ok=%v", err, ok) - } - if ledger.State != MigrationImported || len(ledger.ImportedSessions) != 2 { - t.Fatalf("ledger = %+v", ledger) - } - if _, err := os.Lstat(filepath.Join(f.root, StorageDirName, backupDirName, rawStateBackupName)); err != nil { - t.Fatalf("raw state backup missing: %v", err) - } - // Legacy homes stay in place until validated retirement. - if _, err := os.Lstat(f.legacyRoot("ps-1")); err != nil { - t.Fatalf("legacy home must be preserved after import: %v", err) - } -} - -func TestMigrateHomeConflictFailsClosed(t *testing.T) { - f := newMigrateFixture(t) - f.addSession("ps-1") - f.addSession("ps-2") - seedLegacyHome(t, f.legacyRoot("ps-1"), "shared-index") - seedLegacyHome(t, f.legacyRoot("ps-2"), "shared-index") - // Conflicting relpath: different bytes in each legacy home. - writeContent(t, filepath.Join(f.legacyRoot("ps-1"), "home", ".claude", "projects", "p.json"), "project-A", 0o600) - writeContent(t, filepath.Join(f.legacyRoot("ps-2"), "home", ".claude", "projects", "p.json"), "project-B", 0o600) - - report, err := f.svc.MigrateHome(context.Background(), MigrateHomeOptions{Scope: f.scope, Apply: true}) - if err == nil { - t.Fatalf("conflicting legacy content must fail migration") - } - if len(report.Conflicts) != 1 { - t.Fatalf("conflicts = %+v, want exactly 1", report.Conflicts) - } - conflict := report.Conflicts[0] - if conflict.RelPath != "home/.claude/projects/p.json" { - t.Fatalf("conflict path = %q", conflict.RelPath) - } - for _, key := range f.sessionKeys("ps-1", "ps-2") { - if !contains(conflict.Sessions, key) { - t.Fatalf("conflict sessions %v missing %q", conflict.Sessions, key) - } - } - // No import writes happened. - home := f.sharedHome() - if _, statErr := os.Lstat(filepath.Join(home, "home", ".claude", "projects", "p.json")); !os.IsNotExist(statErr) { - t.Fatalf("conflicting file must not be imported, err=%v", statErr) - } - if _, statErr := os.Lstat(filepath.Join(home, "codex", "sessions", "s1.json")); !os.IsNotExist(statErr) { - t.Fatalf("no file may be imported when any conflict exists, err=%v", statErr) - } - if _, ok, _ := f.svc.RuntimeMigrationLedger(context.Background(), f.scopeHash); ok { - t.Fatalf("conflicted migration must not write the ledger") - } -} - -func TestMigrateHomeDestinationConflictFailsClosed(t *testing.T) { - f := newMigrateFixture(t) - f.addSession("ps-1") - seedLegacyHome(t, f.legacyRoot("ps-1"), "shared-index") - if _, err := f.svc.MigrateHome(context.Background(), MigrateHomeOptions{Scope: f.scope, Apply: true}); err != nil { - t.Fatalf("first MigrateHome: %v", err) - } - // The destination changed underneath (e.g. a concurrent writer): the next - // import of diverging legacy content must fail, never overwrite. - writeContent(t, filepath.Join(f.sharedHome(), "home", ".qoder", "mcp.json"), "locally-modified", 0o600) - report, err := f.svc.MigrateHome(context.Background(), MigrateHomeOptions{Scope: f.scope, Apply: true}) - if err == nil { - t.Fatalf("diverged destination must fail migration") - } - found := false - for _, conflict := range report.Conflicts { - if conflict.RelPath == "home/.qoder/mcp.json" && contains(conflict.Sessions, migrationDestinationMarker) { - found = true - } - } - if !found { - t.Fatalf("destination conflict must name the shared home marker: %+v", report.Conflicts) - } - if got := readContent(t, filepath.Join(f.sharedHome(), "home", ".qoder", "mcp.json")); got != "locally-modified" { - t.Fatalf("destination must not be overwritten, got %q", got) - } -} - -func TestMigrateHomeApplyTwiceIdempotent(t *testing.T) { - f := newMigrateFixture(t) - f.addSession("ps-1") - seedLegacyHome(t, f.legacyRoot("ps-1"), "shared-index") - first, err := f.svc.MigrateHome(context.Background(), MigrateHomeOptions{Scope: f.scope, Apply: true}) - if err != nil { - t.Fatalf("first MigrateHome: %v", err) - } - if first.CopiedFiles != 6 || first.SkippedIdentical != 0 { - t.Fatalf("first run copied=%d skipped=%d, want 6/0", first.CopiedFiles, first.SkippedIdentical) - } - second, err := f.svc.MigrateHome(context.Background(), MigrateHomeOptions{Scope: f.scope, Apply: true}) - if err != nil { - t.Fatalf("second MigrateHome: %v", err) - } - if second.CopiedFiles != 0 || second.SkippedIdentical != 6 { - t.Fatalf("second run copied=%d skipped=%d, want 0/6", second.CopiedFiles, second.SkippedIdentical) - } - if second.LedgerState != MigrationImported { - t.Fatalf("ledger = %q, want imported", second.LedgerState) - } - // The second run found runtime.json on disk and preserved it once. - if _, err := os.Lstat(filepath.Join(f.root, StorageDirName, backupDirName, runtimeBackupName)); err != nil { - t.Fatalf("runtime metadata backup missing after second run: %v", err) - } -} - -func TestMigrateHomeDryRunDoesNotMutate(t *testing.T) { - f := newMigrateFixture(t) - f.addSession("ps-1") - seedLegacyHome(t, f.legacyRoot("ps-1"), "shared-index") - report, err := f.svc.MigrateHome(context.Background(), MigrateHomeOptions{Scope: f.scope, Apply: false}) - if err != nil { - t.Fatalf("MigrateHome dry-run: %v", err) - } - if report.CopiedFiles != 6 { - t.Fatalf("dry-run would-copy = %d, want 6", report.CopiedFiles) - } - if len(report.ImportedSessions) != 1 { - t.Fatalf("dry-run imported sessions = %v", report.ImportedSessions) - } - if _, err := os.Lstat(filepath.Join(f.root, RunnerHomesDirName)); !os.IsNotExist(err) { - t.Fatalf("dry-run must not create %s, err=%v", RunnerHomesDirName, err) - } - if _, ok, _ := f.svc.RuntimeMigrationLedger(context.Background(), f.scopeHash); ok { - t.Fatalf("dry-run must not write the ledger") - } - if _, err := os.Lstat(filepath.Join(f.root, StorageDirName, backupDirName, rawStateBackupName)); !os.IsNotExist(err) { - t.Fatalf("dry-run must not write backups, err=%v", err) - } -} - -func TestMigrateHomeSkipsSessionsWithoutWorkspace(t *testing.T) { - f := newMigrateFixture(t) - f.addSession("ps-1") - seedLegacyHome(t, f.legacyRoot("ps-1"), "shared-index") - // A session without workspace metadata cannot locate its legacy home. - key := state.PublicSessionKey(f.scope.Repo, "ps-3") - f.st.PublicSessions[key] = state.PublicSession{ - Repo: f.scope.Repo, PublicSessionID: "ps-3", AcpxRecordID: "rec-ps-3", Status: state.StatusCompleted, - } - report, err := f.svc.MigrateHome(context.Background(), MigrateHomeOptions{Scope: f.scope, Apply: true}) - if err != nil { - t.Fatalf("MigrateHome: %v", err) - } - if contains(report.ImportedSessions, key) { - t.Fatalf("session without workspace must not import: %v", report.ImportedSessions) - } - joined := strings.Join(report.Diagnostics, "\n") - if !strings.Contains(joined, key) { - t.Fatalf("diagnostics must explain the skipped session: %v", report.Diagnostics) - } -} - -func TestMigrateHomeLedgerNeverRegresses(t *testing.T) { - f := newMigrateFixture(t) - f.addSession("ps-1") - seedLegacyHome(t, f.legacyRoot("ps-1"), "shared-index") - if _, err := f.svc.MigrateHome(context.Background(), MigrateHomeOptions{Scope: f.scope, Apply: true}); err != nil { - t.Fatalf("MigrateHome: %v", err) - } - if err := f.svc.MarkRuntimeMigration(context.Background(), MigrationRecord{ - ScopeHash: f.scopeHash, State: MigrationValidated, ValidatedSession: state.PublicSessionKey(f.scope.Repo, "ps-1"), - }); err != nil { - t.Fatalf("mark validated: %v", err) - } - // Direct regression is rejected. - if err := f.svc.MarkRuntimeMigration(context.Background(), MigrationRecord{ - ScopeHash: f.scopeHash, State: MigrationImported, - }); err == nil { - t.Fatalf("ledger regression must be rejected") - } - // A migration re-run stays idempotent and keeps the advanced ledger state. - report, err := f.svc.MigrateHome(context.Background(), MigrateHomeOptions{Scope: f.scope, Apply: true}) - if err != nil { - t.Fatalf("re-run MigrateHome: %v", err) - } - if report.LedgerState != MigrationValidated { - t.Fatalf("ledger = %q, want validated after re-run", report.LedgerState) - } - ledger, _, err := f.svc.RuntimeMigrationLedger(context.Background(), f.scopeHash) - if err != nil { - t.Fatalf("ledger: %v", err) - } - if ledger.ValidatedSession != state.PublicSessionKey(f.scope.Repo, "ps-1") { - t.Fatalf("validated session lost: %+v", ledger) - } -} - -func TestRetireLegacyHomesRequiresValidated(t *testing.T) { - f := newMigrateFixture(t) - f.addSession("ps-1") - seedLegacyHome(t, f.legacyRoot("ps-1"), "shared-index") - if _, err := f.svc.MigrateHome(context.Background(), MigrateHomeOptions{Scope: f.scope, Apply: true}); err != nil { - t.Fatalf("MigrateHome: %v", err) - } - if _, err := f.svc.RetireLegacyHomes(context.Background(), f.scope, true); err == nil || !strings.Contains(err.Error(), "validated") { - t.Fatalf("retire with imported ledger must require validated, err=%v", err) - } - other := RuntimeScope{Hostname: "host-1", Repo: "o/other", Runner: "runner-1"} - if _, err := f.svc.RetireLegacyHomes(context.Background(), other, true); err == nil { - t.Fatalf("retire without a ledger must fail") - } -} - -func TestRetireLegacyHomes(t *testing.T) { - f := newMigrateFixture(t) - ws1 := f.addSession("ps-1") - ws2 := f.addSession("ps-2") - seedLegacyHome(t, f.legacyRoot("ps-1"), "shared-index") - seedLegacyHome(t, f.legacyRoot("ps-2"), "shared-index") - if _, err := f.svc.MigrateHome(context.Background(), MigrateHomeOptions{Scope: f.scope, Apply: true}); err != nil { - t.Fatalf("MigrateHome: %v", err) - } - if err := f.svc.MarkRuntimeMigration(context.Background(), MigrationRecord{ - ScopeHash: f.scopeHash, State: MigrationValidated, ValidatedSession: state.PublicSessionKey(f.scope.Repo, "ps-1"), - }); err != nil { - t.Fatalf("mark validated: %v", err) - } - for _, sid := range []string{"ps-1", "ps-2"} { - session := f.st.PublicSessions[state.PublicSessionKey(f.scope.Repo, sid)] - if err := f.svc.RecordSessionResources(context.Background(), f.scope.Repo, sid, session.Workspace.Path); err != nil { - t.Fatalf("RecordSessionResources %s: %v", sid, err) - } - } - // The session clones are gone, so the engine can retire the runtimes. - for _, ws := range []string{ws1, ws2} { - if err := os.RemoveAll(ws); err != nil { - t.Fatalf("remove workspace: %v", err) - } - } - - dry, err := f.svc.RetireLegacyHomes(context.Background(), f.scope, false) - if err != nil { - t.Fatalf("dry-run retire: %v", err) - } - if len(dry.RetiredLegacy) != 0 { - t.Fatalf("dry-run must not retire: %+v", dry) - } - for _, sid := range []string{"ps-1", "ps-2"} { - if _, err := os.Lstat(f.legacyRoot(sid)); err != nil { - t.Fatalf("dry-run must not delete legacy home: %v", err) - } - } - - report, err := f.svc.RetireLegacyHomes(context.Background(), f.scope, true) - if err != nil { - t.Fatalf("RetireLegacyHomes: %v", err) - } - if report.LedgerState != MigrationRetired { - t.Fatalf("ledger = %q, want retired", report.LedgerState) - } - for _, key := range f.sessionKeys("ps-1", "ps-2") { - if !contains(report.RetiredLegacy, key) { - t.Fatalf("retired legacy %v missing %q", report.RetiredLegacy, key) - } - } - for _, sid := range []string{"ps-1", "ps-2"} { - if _, err := os.Lstat(f.legacyRoot(sid)); !os.IsNotExist(err) { - t.Fatalf("legacy home for %s must be deleted, err=%v", sid, err) - } - } - // The shared home survives retirement. - if got := readContent(t, filepath.Join(f.sharedHome(), "home", ".acpx", "sessions", "index.json")); got != "shared-index" { - t.Fatalf("shared home damaged by retirement: %q", got) - } - // Idempotent re-run. - second, err := f.svc.RetireLegacyHomes(context.Background(), f.scope, true) - if err != nil { - t.Fatalf("second RetireLegacyHomes: %v", err) - } - if second.LedgerState != MigrationRetired { - t.Fatalf("second run ledger = %q, want retired", second.LedgerState) - } -} - -func TestV1EngineIgnoresRuntimeHomeResources(t *testing.T) { - f := newMigrateFixture(t) - // The upgraded root carries shared homes, job scratch, and runtime.json. - paths, err := PrepareRuntimeHome(f.root, f.scope) - if err != nil { - t.Fatalf("PrepareRuntimeHome: %v", err) - } - if err := f.svc.RecordRuntimeHome(context.Background(), f.scope, paths); err != nil { - t.Fatalf("RecordRuntimeHome: %v", err) - } - scratch, err := PrepareJobScratch(f.root, scratchJobActive) - if err != nil { - t.Fatalf("PrepareJobScratch: %v", err) - } - if err := f.svc.RecordJobScratch(context.Background(), scratchJobActive, scratch.Root); err != nil { - t.Fatalf("RecordJobScratch: %v", err) - } - writeFile(t, filepath.Join(paths.Home, ".acpx", "sessions", "index.json"), 128) - writeFile(t, filepath.Join(scratch.Tmp, "payload"), 64) - // Plus one legacy orphan the v1 engine must still reclaim. - orphanHash := strings.Repeat("ef", 16) - orphanDir := filepath.Join(f.root, SessionsDirName, orphanHash) - writeFile(t, filepath.Join(orphanDir, "home", "stale"), 32) - - engine, err := NewEngine(EngineConfig{ - WorkspaceRoot: f.root, - StateLoader: func(context.Context) (state.RunnerState, error) { return state.NewState(), nil }, - }) - if err != nil { - t.Fatalf("NewEngine: %v", err) - } - if _, err := engine.Reconcile(context.Background(), ReconcileOptions{Apply: true, OrphanGrace: 0}); err != nil { - t.Fatalf("first Reconcile: %v", err) - } - report, err := engine.Reconcile(context.Background(), ReconcileOptions{Apply: true, OrphanGrace: 0}) - if err != nil { - t.Fatalf("second Reconcile: %v", err) - } - orphanID := ResourceID(ResourceKindSessionRuntime, "", "", orphanHash) - got := reportByID(t, report, orphanID) - if got.Action != ActionDeleted { - t.Fatalf("v1 engine must still delete the legacy orphan, got %+v", got) - } - if _, err := os.Lstat(orphanDir); !os.IsNotExist(err) { - t.Fatalf("legacy orphan must be gone, err=%v", err) - } - // The v1 engine neither inventories nor deletes the new roots. - for _, resource := range report.Resources { - if strings.Contains(resource.ID, RunnerHomesDirName) || strings.Contains(resource.ID, JobScratchDirName) || - strings.Contains(resource.Hash, RunnerHomesDirName) { - t.Fatalf("v1 engine must never inventory runner-home resources: %+v", resource) - } - } - if _, err := os.Lstat(filepath.Join(paths.Home, ".acpx", "sessions", "index.json")); err != nil { - t.Fatalf("shared runtime home damaged by v1 engine: %v", err) - } - if _, err := os.Lstat(filepath.Join(scratch.Tmp, "payload")); err != nil { - t.Fatalf("job scratch damaged by v1 engine: %v", err) - } - // The sidecar stays kind-valid and runtime.json is untouched. - store, err := OpenStore(f.root) - if err != nil { - t.Fatalf("OpenStore: %v", err) - } - defer store.Close() - for id, resource := range store.State().Resources { - if !resource.Kind.Valid() { - t.Fatalf("sidecar resource %q has invalid kind %q", id, resource.Kind) - } - } - runtimeStore, err := OpenRuntimeStore(f.root) - if err != nil { - t.Fatalf("OpenRuntimeStore: %v", err) - } - defer runtimeStore.Close() - runtimeState := runtimeStore.State() - if _, ok := runtimeState.Homes[f.scopeHash]; !ok { - t.Fatalf("runtime home record lost: %+v", runtimeState.Homes) - } - if _, ok := runtimeState.Scratch[scratchJobActive]; !ok { - t.Fatalf("job scratch record lost: %+v", runtimeState.Scratch) - } -} diff --git a/internal/commentrunner/storage/runtime_reconcile.go b/internal/commentrunner/storage/runtime_reconcile.go index 5921ec10..13ecc40b 100644 --- a/internal/commentrunner/storage/runtime_reconcile.go +++ b/internal/commentrunner/storage/runtime_reconcile.go @@ -108,44 +108,46 @@ func (s *Service) ReconcileJobScratch(ctx context.Context, apply bool) (RuntimeR active[job.ID] = true } } - store, err := s.RuntimeStore() - if err != nil { - return RuntimeReconcileReport{}, err - } - if err := store.Reload(); err != nil { - return RuntimeReconcileReport{}, fmt.Errorf("reload runtime metadata: %w", err) + if err := s.store.Reload(); err != nil { + return RuntimeReconcileReport{}, fmt.Errorf("reload storage sidecar: %w", err) } report := RuntimeReconcileReport{} mutate := apply - if store.Status() == SidecarReportOnly { + if s.store.Status() == SidecarReportOnly { mutate = false - report.Diagnostics = append(report.Diagnostics, "runtime metadata is report-only: foreign root identity or newer schema; inventory only, no mutations") + report.Diagnostics = append(report.Diagnostics, "storage sidecar is report-only: foreign root identity or newer schema; inventory only, no mutations") } - if cause := store.LoadCause(); cause != nil { - report.Diagnostics = append(report.Diagnostics, safeDiagnostic("runtime metadata rebuilt after corruption ("+cause.Err.Error()+")")) + if cause := s.store.LoadCause(); cause != nil { + report.Diagnostics = append(report.Diagnostics, safeDiagnostic("storage sidecar rebuilt after corruption ("+cause.Err.Error()+")")) } if !apply { report.Diagnostics = append(report.Diagnostics, "dry-run: no mutations performed") } - recorded := store.State().Scratch + recorded := map[string]PhysicalResource{} + for id, resource := range s.store.State().Resources { + if resource.Kind == ResourceKindJobScratch { + recorded[id] = resource + } + } recordedIDs := make([]string, 0, len(recorded)) - for jobID := range recorded { - recordedIDs = append(recordedIDs, jobID) + for id := range recorded { + recordedIDs = append(recordedIDs, id) } sort.Strings(recordedIDs) - for _, jobID := range recordedIDs { - record := recorded[jobID] + for _, id := range recordedIDs { + record := recorded[id] + jobID := record.PhysicalHash if active[jobID] { report.ScratchKept = append(report.ScratchKept, jobID) if mutate && record.CleanupState != CleanupManaged { // A crash-interrupted completion raced a still-active job: // heal the record instead of deleting live scratch. - if err := store.Update(func(st *RuntimeState) error { - current, ok := st.Scratch[jobID] + if err := s.store.Update(func(st *StorageState) error { + current, ok := st.Resources[id] if ok { current.CleanupState = CleanupManaged - st.Scratch[jobID] = current + st.Resources[id] = current } return nil }); err != nil { @@ -157,8 +159,8 @@ func (s *Service) ReconcileJobScratch(ctx context.Context, apply bool) (RuntimeR if _, statErr := os.Lstat(record.Path); errors.Is(statErr, os.ErrNotExist) { // Directory already gone: garbage-collect the record. if mutate { - if err := store.Update(func(st *RuntimeState) error { - delete(st.Scratch, jobID) + if err := s.store.Update(func(st *StorageState) error { + delete(st.Resources, id) return nil }); err != nil { report.Diagnostics = append(report.Diagnostics, safeDiagnostic("job scratch "+jobID+" record GC failed: "+err.Error())) @@ -174,16 +176,20 @@ func (s *Service) ReconcileJobScratch(ctx context.Context, apply bool) (RuntimeR report.Diagnostics = append(report.Diagnostics, "would remove job scratch "+jobID) continue } - if err := s.removeJobScratch(store, record, &report); err != nil { + if err := s.removeJobScratch(record, &report); err != nil { report.Diagnostics = append(report.Diagnostics, safeDiagnostic("job scratch "+jobID+" removal failed: "+err.Error())) } } - // On-disk entries with no metadata record. + // On-disk entries with no sidecar record. + recordedJobs := map[string]bool{} + for _, record := range recorded { + recordedJobs[record.PhysicalHash] = true + } names, err := readDirNames(filepath.Join(s.root, JobScratchDirName)) if err == nil { for _, name := range names { - if _, ok := recorded[name]; ok { + if recordedJobs[name] { continue } if !jobScratchIDPattern.MatchString(name) { @@ -221,14 +227,14 @@ func (s *Service) ReconcileJobScratch(ctx context.Context, apply bool) (RuntimeR // removeJobScratch runs the recorded-entry lifecycle: mark deleting, remove // the directory capability scoped, then garbage-collect the record. -func (s *Service) removeJobScratch(store *RuntimeStore, record JobScratchRecord, report *RuntimeReconcileReport) error { - if err := store.Update(func(st *RuntimeState) error { - current, ok := st.Scratch[record.JobID] +func (s *Service) removeJobScratch(record PhysicalResource, report *RuntimeReconcileReport) error { + if err := s.store.Update(func(st *StorageState) error { + current, ok := st.Resources[record.ID] if !ok { return nil } current.CleanupState = CleanupDeleting - st.Scratch[record.JobID] = current + st.Resources[record.ID] = current return nil }); err != nil { return err @@ -237,13 +243,13 @@ func (s *Service) removeJobScratch(store *RuntimeStore, record JobScratchRecord, if err := removeOpenedTree(record.Path, nil); err != nil { return err } - if err := store.Update(func(st *RuntimeState) error { - delete(st.Scratch, record.JobID) + if err := s.store.Update(func(st *StorageState) error { + delete(st.Resources, record.ID) return nil }); err != nil { return err } - report.ScratchRemoved = append(report.ScratchRemoved, record.JobID) + report.ScratchRemoved = append(report.ScratchRemoved, record.PhysicalHash) report.ReclaimedBytes += measured return nil } @@ -257,29 +263,30 @@ func (s *Service) EvictRuntimeCaches(ctx context.Context, apply bool) (RuntimeRe return RuntimeReconcileReport{}, fmt.Errorf("runtime cache eviction owner: %w", err) } defer release() - store, err := s.RuntimeStore() - if err != nil { - return RuntimeReconcileReport{}, err - } - if err := store.Reload(); err != nil { - return RuntimeReconcileReport{}, fmt.Errorf("reload runtime metadata: %w", err) + if err := s.store.Reload(); err != nil { + return RuntimeReconcileReport{}, fmt.Errorf("reload storage sidecar: %w", err) } report := RuntimeReconcileReport{} mutate := apply - if store.Status() == SidecarReportOnly { + if s.store.Status() == SidecarReportOnly { mutate = false - report.Diagnostics = append(report.Diagnostics, "runtime metadata is report-only: foreign root identity or newer schema; inventory only, no mutations") + report.Diagnostics = append(report.Diagnostics, "storage sidecar is report-only: foreign root identity or newer schema; inventory only, no mutations") + } + homes := map[string]PhysicalResource{} + for id, resource := range s.store.State().Resources { + if resource.Kind == ResourceKindRunnerHome { + homes[id] = resource + } } - homes := store.State().Homes - hashes := make([]string, 0, len(homes)) - for hash := range homes { - hashes = append(hashes, hash) + ids := make([]string, 0, len(homes)) + for id := range homes { + ids = append(ids, id) } - sort.Strings(hashes) - for _, hash := range hashes { - home := homes[hash] + sort.Strings(ids) + for _, id := range ids { + home := homes[id] if err := s.validateRuntimeHomeRecord(home); err != nil { - report.Diagnostics = append(report.Diagnostics, safeDiagnostic("runtime home "+hash+" record fails validation: "+err.Error())) + report.Diagnostics = append(report.Diagnostics, safeDiagnostic("runtime home "+home.PhysicalHash+" record fails validation: "+err.Error())) continue } homeDir := RuntimeHomePathsFor(home.Path).Home @@ -315,11 +322,11 @@ func (s *Service) EvictRuntimeCaches(ctx context.Context, apply bool) (RuntimeRe // validateRuntimeHomeRecord proves a recorded home still names the canonical // scoped path below this root before any of its subtrees are touched. -func (s *Service) validateRuntimeHomeRecord(home RuntimeHomeRecord) error { - if !ValidHashName(home.Hash) { - return fmt.Errorf("home hash %q is invalid", home.Hash) +func (s *Service) validateRuntimeHomeRecord(home PhysicalResource) error { + if !ValidHashName(home.PhysicalHash) { + return fmt.Errorf("home hash %q is invalid", home.PhysicalHash) } - expected := filepath.Join(s.root, RunnerHomesDirName, home.Hash) + expected := filepath.Join(s.root, RunnerHomesDirName, home.PhysicalHash) if filepath.Clean(home.Path) != expected { return fmt.Errorf("home path %q does not match the scoped home %q for this root", home.Path, expected) } diff --git a/internal/commentrunner/storage/runtime_reconcile_test.go b/internal/commentrunner/storage/runtime_reconcile_test.go index 1cf78210..a85f3d08 100644 --- a/internal/commentrunner/storage/runtime_reconcile_test.go +++ b/internal/commentrunner/storage/runtime_reconcile_test.go @@ -42,6 +42,16 @@ func prepareScratchWithFile(t *testing.T, root, jobID string, size int) JobScrat return paths } +func scratchRecordID(repo, jobID string) string { + return ResourceID(ResourceKindJobScratch, repo, "", jobID) +} + +func scratchRecord(t *testing.T, svc *Service, repo, jobID string) (PhysicalResource, bool) { + t.Helper() + record, ok := svc.Store().State().Resources[scratchRecordID(repo, jobID)] + return record, ok +} + func contains(values []string, needle string) bool { for _, value := range values { if value == needle { @@ -56,7 +66,7 @@ func TestReconcileJobScratchKeepsActiveJob(t *testing.T) { st.Jobs[scratchJobActive] = state.Job{ID: scratchJobActive, Repo: "o/r", Status: state.StatusRunning} svc, root := newRuntimeService(t, st) paths := prepareScratchWithFile(t, root, scratchJobActive, 128) - if err := svc.RecordJobScratch(context.Background(), scratchJobActive, paths.Root); err != nil { + if err := svc.RecordJobScratch(context.Background(), "o/r", scratchJobActive, paths.Root); err != nil { t.Fatalf("RecordJobScratch: %v", err) } report, err := svc.ReconcileJobScratch(context.Background(), true) @@ -72,12 +82,9 @@ func TestReconcileJobScratchKeepsActiveJob(t *testing.T) { if _, err := os.Lstat(paths.Root); err != nil { t.Fatalf("active job scratch dir removed: %v", err) } - store, err := svc.RuntimeStore() - if err != nil { - t.Fatalf("RuntimeStore: %v", err) - } - if record := store.State().Scratch[scratchJobActive]; record.CleanupState != CleanupManaged { - t.Fatalf("record cleanup state = %q, want managed", record.CleanupState) + record, ok := scratchRecord(t, svc, "o/r", scratchJobActive) + if !ok || record.CleanupState != CleanupManaged { + t.Fatalf("record = %+v ok=%v, want managed", record, ok) } // Interrupted jobs are still active for scratch purposes. st.Jobs[scratchJobActive] = state.Job{ID: scratchJobActive, Repo: "o/r", Status: state.StatusInterrupted} @@ -97,7 +104,7 @@ func TestReconcileJobScratchRemovesTerminalAndUnknown(t *testing.T) { donePaths := prepareScratchWithFile(t, root, scratchJobDone, 100) orphanPaths := prepareScratchWithFile(t, root, scratchJobOrphan, 50) for jobID, paths := range map[string]JobScratchPaths{scratchJobDone: donePaths, scratchJobOrphan: orphanPaths} { - if err := svc.RecordJobScratch(context.Background(), jobID, paths.Root); err != nil { + if err := svc.RecordJobScratch(context.Background(), "o/r", jobID, paths.Root); err != nil { t.Fatalf("RecordJobScratch %s: %v", jobID, err) } } @@ -116,12 +123,10 @@ func TestReconcileJobScratchRemovesTerminalAndUnknown(t *testing.T) { t.Fatalf("scratch dir %q must be gone, err=%v", paths.Root, err) } } - store, err := svc.RuntimeStore() - if err != nil { - t.Fatalf("RuntimeStore: %v", err) - } - if len(store.State().Scratch) != 0 { - t.Fatalf("scratch records must be garbage-collected: %+v", store.State().Scratch) + for id, resource := range svc.Store().State().Resources { + if resource.Kind == ResourceKindJobScratch { + t.Fatalf("scratch records must be garbage-collected: %q = %+v", id, resource) + } } // Second apply is a no-op. second, err := svc.ReconcileJobScratch(context.Background(), true) @@ -135,7 +140,7 @@ func TestReconcileJobScratchRemovesTerminalAndUnknown(t *testing.T) { func TestReconcileJobScratchRemovesUnrecordedOnDiskEntry(t *testing.T) { svc, root := newRuntimeService(t, state.NewState()) - // A crashed runner left scratch behind without a metadata record. + // A crashed runner left scratch behind without a sidecar record. paths := prepareScratchWithFile(t, root, scratchJobOrphan, 64) report, err := svc.ReconcileJobScratch(context.Background(), true) if err != nil { @@ -178,7 +183,7 @@ func TestReconcileJobScratchDryRunDoesNotMutate(t *testing.T) { st.Jobs[scratchJobDryRun] = state.Job{ID: scratchJobDryRun, Repo: "o/r", Status: state.StatusFailed} svc, root := newRuntimeService(t, st) paths := prepareScratchWithFile(t, root, scratchJobDryRun, 32) - if err := svc.RecordJobScratch(context.Background(), scratchJobDryRun, paths.Root); err != nil { + if err := svc.RecordJobScratch(context.Background(), "o/r", scratchJobDryRun, paths.Root); err != nil { t.Fatalf("RecordJobScratch: %v", err) } report, err := svc.ReconcileJobScratch(context.Background(), false) @@ -194,54 +199,77 @@ func TestReconcileJobScratchDryRunDoesNotMutate(t *testing.T) { if _, err := os.Lstat(paths.Root); err != nil { t.Fatalf("dry-run must not delete: %v", err) } - store, err := svc.RuntimeStore() - if err != nil { - t.Fatalf("RuntimeStore: %v", err) - } - if record := store.State().Scratch[scratchJobDryRun]; record.CleanupState != CleanupManaged { - t.Fatalf("dry-run must not mutate records, state = %q", record.CleanupState) + record, ok := scratchRecord(t, svc, "o/r", scratchJobDryRun) + if !ok || record.CleanupState != CleanupManaged { + t.Fatalf("dry-run must not mutate records, record = %+v ok=%v", record, ok) } } func TestCompleteJobScratch(t *testing.T) { svc, root := newRuntimeService(t, state.NewState()) paths := prepareScratchWithFile(t, root, scratchJobDone, 16) - if err := svc.RecordJobScratch(context.Background(), scratchJobDone, paths.Root); err != nil { + if err := svc.RecordJobScratch(context.Background(), "o/r", scratchJobDone, paths.Root); err != nil { t.Fatalf("RecordJobScratch: %v", err) } - if err := svc.CompleteJobScratch(context.Background(), scratchJobDone); err != nil { + if err := svc.CompleteJobScratch(context.Background(), "o/r", scratchJobDone); err != nil { t.Fatalf("CompleteJobScratch: %v", err) } if _, err := os.Lstat(paths.Root); !os.IsNotExist(err) { t.Fatalf("scratch dir must be gone, err=%v", err) } - store, err := svc.RuntimeStore() - if err != nil { - t.Fatalf("RuntimeStore: %v", err) - } - if len(store.State().Scratch) != 0 { - t.Fatalf("record must be garbage-collected: %+v", store.State().Scratch) + if _, ok := scratchRecord(t, svc, "o/r", scratchJobDone); ok { + t.Fatalf("record must be garbage-collected") } // Idempotent and tolerant of unknown jobs. - if err := svc.CompleteJobScratch(context.Background(), scratchJobDone); err != nil { + if err := svc.CompleteJobScratch(context.Background(), "o/r", scratchJobDone); err != nil { t.Fatalf("second CompleteJobScratch: %v", err) } - if err := svc.CompleteJobScratch(context.Background(), scratchJobOrphan); err != nil { + if err := svc.CompleteJobScratch(context.Background(), "o/r", scratchJobOrphan); err != nil { t.Fatalf("unknown job must be a no-op: %v", err) } - if err := svc.CompleteJobScratch(context.Background(), "job-1"); err == nil { + if err := svc.CompleteJobScratch(context.Background(), "o/r", "job-1"); err == nil { t.Fatalf("invalid job id must be rejected") } } func TestRecordJobScratchRejectsForeignPath(t *testing.T) { svc, root := newRuntimeService(t, state.NewState()) - if err := svc.RecordJobScratch(context.Background(), scratchJobDone, filepath.Join(root, "elsewhere", scratchJobDone)); err == nil { + if err := svc.RecordJobScratch(context.Background(), "o/r", scratchJobDone, filepath.Join(root, "elsewhere", scratchJobDone)); err == nil { t.Fatalf("recording a scratch path outside .job-scratch must fail closed") } - if err := svc.RecordJobScratch(context.Background(), "job-1", filepath.Join(root, JobScratchDirName, "job-1")); err == nil { + if err := svc.RecordJobScratch(context.Background(), "o/r", "job-1", filepath.Join(root, JobScratchDirName, "job-1")); err == nil { t.Fatalf("invalid job id must be rejected") } + if err := svc.RecordJobScratch(context.Background(), "", scratchJobDone, filepath.Join(root, JobScratchDirName, scratchJobDone)); err == nil { + t.Fatalf("empty repo must be rejected") + } +} + +func TestRecordJobScratchUpsertIdempotent(t *testing.T) { + svc, root := newRuntimeService(t, state.NewState()) + paths := prepareScratchWithFile(t, root, scratchJobDone, 8) + if err := svc.RecordJobScratch(context.Background(), "o/r", scratchJobDone, paths.Root); err != nil { + t.Fatalf("RecordJobScratch: %v", err) + } + first, ok := scratchRecord(t, svc, "o/r", scratchJobDone) + if !ok { + t.Fatalf("scratch record missing") + } + if first.Kind != ResourceKindJobScratch || first.Repo != "o/r" || first.PublicSessionID != "" || + first.PhysicalHash != scratchJobDone || first.Path != paths.Root || first.CleanupState != CleanupManaged { + t.Fatalf("scratch record = %+v", first) + } + if first.FirstObservedAt.IsZero() { + t.Fatalf("scratch record must carry first observation proof") + } + // A steady-state re-record preserves the observation proof and the record. + if err := svc.RecordJobScratch(context.Background(), "o/r", scratchJobDone, paths.Root); err != nil { + t.Fatalf("re-record: %v", err) + } + second, ok := scratchRecord(t, svc, "o/r", scratchJobDone) + if !ok || second != first { + t.Fatalf("idempotent re-record changed the record: first=%+v second=%+v", first, second) + } } func TestEvictRuntimeCaches(t *testing.T) { @@ -358,18 +386,15 @@ func TestEvictRuntimeCachesRejectsForeignHomePath(t *testing.T) { t.Fatalf("RecordRuntimeHome: %v", err) } // Tamper: repoint the recorded home outside the root. - store, err := svc.RuntimeStore() - if err != nil { - t.Fatalf("RuntimeStore: %v", err) - } hash, err := RuntimeScopeHash(scope) if err != nil { t.Fatalf("hash: %v", err) } - if err := store.Update(func(st *RuntimeState) error { - record := st.Homes[hash] + id := ResourceID(ResourceKindRunnerHome, scope.Repo, "", hash) + if err := svc.Store().Update(func(st *StorageState) error { + record := st.Resources[id] record.Path = filepath.Join(root, ".sessions", strings.Repeat("de", 16)) - st.Homes[hash] = record + st.Resources[id] = record return nil }); err != nil { t.Fatalf("tamper update: %v", err) diff --git a/internal/commentrunner/storage/runtime_sidecar_test.go b/internal/commentrunner/storage/runtime_sidecar_test.go new file mode 100644 index 00000000..eb1a693f --- /dev/null +++ b/internal/commentrunner/storage/runtime_sidecar_test.go @@ -0,0 +1,290 @@ +package storage + +import ( + "context" + "errors" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/higress-group/issue-spec/internal/commentrunner/state" +) + +func TestRuntimeResourceKindsValid(t *testing.T) { + for _, kind := range []ResourceKind{ + ResourceKindSessionRuntime, ResourceKindSessionProcessPool, + ResourceKindRunnerHome, ResourceKindJobScratch, + } { + if !kind.Valid() { + t.Fatalf("kind %q must be valid", kind) + } + } + if ResourceKind("runtime_home").Valid() || ResourceKind("").Valid() { + t.Fatalf("unknown kinds must stay invalid") + } +} + +func TestRecordRuntimeHomeUpsertIdempotent(t *testing.T) { + svc, root := newRuntimeService(t, state.NewState()) + scope := testScope() + paths, err := PrepareRuntimeHome(root, scope) + if err != nil { + t.Fatalf("PrepareRuntimeHome: %v", err) + } + if err := svc.RecordRuntimeHome(context.Background(), scope, paths); err != nil { + t.Fatalf("RecordRuntimeHome: %v", err) + } + hash, err := RuntimeScopeHash(scope) + if err != nil { + t.Fatalf("hash: %v", err) + } + id := ResourceID(ResourceKindRunnerHome, scope.Repo, "", hash) + first, ok := svc.Store().State().Resources[id] + if !ok { + t.Fatalf("runtime home record missing") + } + if first.Kind != ResourceKindRunnerHome || first.Repo != scope.Repo || first.PublicSessionID != "" || + first.PhysicalHash != hash || first.Path != paths.Root || first.CleanupState != CleanupManaged { + t.Fatalf("runtime home record = %+v", first) + } + if first.FirstObservedAt.IsZero() { + t.Fatalf("runtime home record must carry first observation proof") + } + // A steady-state re-record preserves the observation proof and the record. + if err := svc.RecordRuntimeHome(context.Background(), scope, paths); err != nil { + t.Fatalf("re-record: %v", err) + } + second, ok := svc.Store().State().Resources[id] + if !ok || second != first { + t.Fatalf("idempotent re-record changed the record: first=%+v second=%+v", first, second) + } + // A foreign home path fails closed. + if err := svc.RecordRuntimeHome(context.Background(), scope, RuntimeHomePaths{Root: filepath.Join(root, "elsewhere")}); err == nil { + t.Fatalf("recording a home outside .runner-home must fail closed") + } +} + +// TestEngineSweepDropsStaleJobScratchRecord: a job_scratch record whose +// directory vanished outside reconciliation is dropped with a diagnostic by +// the generic engine pass, while sibling records stay intact. +func TestEngineSweepDropsStaleJobScratchRecord(t *testing.T) { + svc, root := newRuntimeService(t, state.NewState()) + stalePaths := prepareScratchWithFile(t, root, scratchJobDone, 16) + livePaths := prepareScratchWithFile(t, root, scratchJobActive, 16) + if err := svc.RecordJobScratch(context.Background(), "o/r", scratchJobDone, stalePaths.Root); err != nil { + t.Fatalf("RecordJobScratch stale: %v", err) + } + if err := svc.RecordJobScratch(context.Background(), "o/r", scratchJobActive, livePaths.Root); err != nil { + t.Fatalf("RecordJobScratch live: %v", err) + } + if err := os.RemoveAll(stalePaths.Root); err != nil { + t.Fatalf("remove stale scratch: %v", err) + } + report, err := svc.ReconcileStorage(context.Background(), true, false) + if err != nil { + t.Fatalf("ReconcileStorage: %v", err) + } + if _, ok := scratchRecord(t, svc, "o/r", scratchJobDone); ok { + t.Fatalf("stale scratch record must be swept") + } + found := false + for _, diagnostic := range report.Diagnostics { + if strings.Contains(diagnostic, scratchJobDone) && strings.Contains(diagnostic, "vanished") { + found = true + } + } + if !found { + t.Fatalf("sweep must diagnose the vanished record: %+v", report.Diagnostics) + } + live, ok := scratchRecord(t, svc, "o/r", scratchJobActive) + if !ok || live.CleanupState != CleanupManaged { + t.Fatalf("live scratch record must survive the sweep: %+v ok=%v", live, ok) + } + if _, err := os.Lstat(livePaths.Root); err != nil { + t.Fatalf("live scratch dir must survive the sweep: %v", err) + } +} + +// TestRunnerHomeSurvivesApplyReconcile is the forward-looking engine contract: +// the engine never inventories `.runner-home`/`.job-scratch`, so a runner_home +// record and its tree survive an apply reconcile while a legacy `.sessions` +// orphan is still reclaimed. +func TestRunnerHomeSurvivesApplyReconcile(t *testing.T) { + svc, root := newRuntimeService(t, state.NewState()) + scope := testScope() + paths, err := PrepareRuntimeHome(root, scope) + if err != nil { + t.Fatalf("PrepareRuntimeHome: %v", err) + } + if err := svc.RecordRuntimeHome(context.Background(), scope, paths); err != nil { + t.Fatalf("RecordRuntimeHome: %v", err) + } + scratch := prepareScratchWithFile(t, root, scratchJobActive, 64) + if err := svc.RecordJobScratch(context.Background(), "o/r", scratchJobActive, scratch.Root); err != nil { + t.Fatalf("RecordJobScratch: %v", err) + } + writeFile(t, filepath.Join(paths.Home, ".acpx", "sessions", "index.json"), 128) + // Plus one legacy orphan the engine must still reclaim. + orphanHash := strings.Repeat("ef", 16) + orphanDir := filepath.Join(root, SessionsDirName, orphanHash) + writeFile(t, filepath.Join(orphanDir, "home", "stale"), 32) + + orphanID := ResourceID(ResourceKindSessionRuntime, "", "", orphanHash) + var report Report + // Two passes: the first observes the orphan, the second reclaims it past + // a zero grace window. + for pass := 0; pass < 2; pass++ { + report, err = svc.ReconcileStorage(context.Background(), true, false) + if err != nil { + t.Fatalf("ReconcileStorage pass %d: %v", pass, err) + } + svc.orphanGrace = 0 + } + got := reportByID(t, report, orphanID) + if got.Action != ActionDeleted { + t.Fatalf("engine must still delete the legacy orphan, got %+v", got) + } + if _, err := os.Lstat(orphanDir); !os.IsNotExist(err) { + t.Fatalf("legacy orphan must be gone, err=%v", err) + } + // The engine neither inventories nor schedules the shared-layout resources. + for _, resource := range report.Resources { + if resource.Kind == ResourceKindRunnerHome || resource.Kind == ResourceKindJobScratch { + t.Fatalf("engine must never classify shared-layout resources: %+v", resource) + } + if strings.Contains(resource.ID, RunnerHomesDirName) || strings.Contains(resource.ID, JobScratchDirName) { + t.Fatalf("engine must never inventory runner-home resources: %+v", resource) + } + } + hash, err := RuntimeScopeHash(scope) + if err != nil { + t.Fatalf("hash: %v", err) + } + homeRecord, ok := svc.Store().State().Resources[ResourceID(ResourceKindRunnerHome, scope.Repo, "", hash)] + if !ok || homeRecord.CleanupState != CleanupManaged { + t.Fatalf("runtime home record must survive reconcile: %+v ok=%v", homeRecord, ok) + } + if _, ok := scratchRecord(t, svc, "o/r", scratchJobActive); !ok { + t.Fatalf("job scratch record must survive reconcile") + } + if _, err := os.Lstat(filepath.Join(paths.Home, ".acpx", "sessions", "index.json")); err != nil { + t.Fatalf("shared runtime home damaged by the engine: %v", err) + } + if _, err := os.Lstat(filepath.Join(scratch.Root, "tmp", "payload")); err != nil { + t.Fatalf("job scratch damaged by the engine: %v", err) + } +} + +// TestAdmitDispatchPressuredReclaimsScratchAndCaches: the pressured pass also +// reclaims stale job scratch and rebuildable home caches once the root runs +// the shared layout. +func TestAdmitDispatchPressuredReclaimsScratchAndCaches(t *testing.T) { + svc, root := newRuntimeService(t, state.NewState()) + svc.minFreeBytes = 1024 + scope := testScope() + paths, err := PrepareRuntimeHome(root, scope) + if err != nil { + t.Fatalf("PrepareRuntimeHome: %v", err) + } + if err := svc.RecordRuntimeHome(context.Background(), scope, paths); err != nil { + t.Fatalf("RecordRuntimeHome: %v", err) + } + writeFile(t, filepath.Join(paths.Home, ".cache", "go-build", "abc"), 256) + scratch := prepareScratchWithFile(t, root, scratchJobDone, 128) + if err := svc.RecordJobScratch(context.Background(), "o/r", scratchJobDone, scratch.Root); err != nil { + t.Fatalf("RecordJobScratch: %v", err) + } + calls := 0 + svc.statfs = func(string) (uint64, error) { + calls++ + if calls == 1 { + return 512, nil + } + return 4096, nil + } + if err := svc.AdmitDispatch(context.Background()); err != nil { + t.Fatalf("AdmitDispatch after pressured reclaim: %v", err) + } + if calls != 2 { + t.Fatalf("statfs calls=%d, want 2 (pressure check + post-cleanup recheck)", calls) + } + if _, err := os.Lstat(filepath.Join(paths.Home, ".cache")); !os.IsNotExist(err) { + t.Fatalf("pressured admission must evict rebuildable caches, err=%v", err) + } + if _, err := os.Lstat(scratch.Root); !os.IsNotExist(err) { + t.Fatalf("pressured admission must remove stale job scratch, err=%v", err) + } + if _, ok := scratchRecord(t, svc, "o/r", scratchJobDone); ok { + t.Fatalf("stale scratch record must be gone after pressured reclaim") + } + // Protected home content survives. + if _, err := os.Lstat(filepath.Join(paths.Root, runtimeScopeFileName)); err != nil { + t.Fatalf("scope binding must survive pressured reclaim: %v", err) + } +} + +// TestAdmitDispatchPressuredSkipsSharedCleanupWithoutHomes: without any +// runner_home record the pressured pass leaves the shared-layout directories +// alone, so a root that never adopted the layout sees no new cleanup behavior. +func TestAdmitDispatchPressuredSkipsSharedCleanupWithoutHomes(t *testing.T) { + svc, root := newRuntimeService(t, state.NewState()) + svc.minFreeBytes = 1024 + scratch := prepareScratchWithFile(t, root, scratchJobDone, 128) + if err := svc.RecordJobScratch(context.Background(), "o/r", scratchJobDone, scratch.Root); err != nil { + t.Fatalf("RecordJobScratch: %v", err) + } + calls := 0 + svc.statfs = func(string) (uint64, error) { + calls++ + if calls == 1 { + return 512, nil + } + return 4096, nil + } + if err := svc.AdmitDispatch(context.Background()); err != nil { + t.Fatalf("AdmitDispatch: %v", err) + } + if _, err := os.Lstat(scratch.Root); err != nil { + t.Fatalf("scratch must be untouched without a runner home record: %v", err) + } +} + +// TestAdmitDispatchPressuredCleanupFailureIsBestEffort: a failing shared +// cleanup never fails admission by itself; it surfaces only as a bounded +// diagnostic while pressure persists. +func TestAdmitDispatchPressuredCleanupFailureIsBestEffort(t *testing.T) { + svc, root := newRuntimeService(t, state.NewState()) + svc.minFreeBytes = 1024 + scope := testScope() + paths, err := PrepareRuntimeHome(root, scope) + if err != nil { + t.Fatalf("PrepareRuntimeHome: %v", err) + } + if err := svc.RecordRuntimeHome(context.Background(), scope, paths); err != nil { + t.Fatalf("RecordRuntimeHome: %v", err) + } + // Break only the scratch reconciliation's state reload: the pressured + // engine pass loads state first and must succeed. + loaderCalls := 0 + svc.stateLoader = func(context.Context) (state.RunnerState, error) { + loaderCalls++ + if loaderCalls > 1 { + return state.RunnerState{}, errors.New("state unavailable") + } + return state.NewState(), nil + } + // First statfs below threshold, then recovered: admission passes despite + // the best-effort cleanup failure. + calls := 0 + svc.statfs = func(string) (uint64, error) { + calls++ + if calls == 1 { + return 512, nil + } + return 4096, nil + } + if err := svc.AdmitDispatch(context.Background()); err != nil { + t.Fatalf("best-effort cleanup failure must not fail admission: %v", err) + } +} diff --git a/internal/commentrunner/storage/runtime_store.go b/internal/commentrunner/storage/runtime_store.go deleted file mode 100644 index be17e5da..00000000 --- a/internal/commentrunner/storage/runtime_store.go +++ /dev/null @@ -1,387 +0,0 @@ -package storage - -import ( - "bytes" - "encoding/json" - "errors" - "fmt" - "io" - "os" - "path/filepath" - "strings" - "sync" - "time" - - "github.com/higress-group/issue-spec/internal/commentrunner/state" -) - -// RuntimeSchemaVersion is the current `.storage/runtime.json` schema. A newer -// on-disk schema or a foreign root identity permits report-only inventory, -// exactly like the sidecar. The runtime metadata lives in its own file so a -// v1 binary never parses records it cannot classify. -const RuntimeSchemaVersion = 1 - -const ( - runtimeFileName = "runtime.json" - runtimeLockName = "runtime.json.lock" - runtimeCorruptBackup = "runtime-corrupt-latest.json" -) - -// MigrationState is the ledger lifecycle of one scope's legacy-home import: -// imported -> validated -> retired. Transitions are monotonic. -type MigrationState string - -const ( - MigrationImported MigrationState = "imported" - MigrationValidated MigrationState = "validated" - MigrationRetired MigrationState = "retired" -) - -func (s MigrationState) Valid() bool { - switch s { - case MigrationImported, MigrationValidated, MigrationRetired: - return true - default: - return false - } -} - -// migrationStateRank orders ledger states; regressions are rejected. -func migrationStateRank(s MigrationState) int { - switch s { - case MigrationImported: - return 1 - case MigrationValidated: - return 2 - case MigrationRetired: - return 3 - default: - return 0 - } -} - -// RuntimeHomeRecord pins one scope hash to its prepared runtime home. -type RuntimeHomeRecord struct { - Hash string `json:"hash"` - Path string `json:"path"` - Hostname string `json:"hostname"` - Realm string `json:"realm,omitempty"` - Repo string `json:"repo"` - Runner string `json:"runner"` - CreatedAt time.Time `json:"created_at,omitempty"` -} - -// JobScratchRecord tracks one job's scratch directory through the same -// managed/deleting/removed cleanup lifecycle as sidecar resources. -type JobScratchRecord struct { - JobID string `json:"job_id"` - Path string `json:"path"` - CreatedAt time.Time `json:"created_at,omitempty"` - CleanupState CleanupState `json:"cleanup_state,omitempty"` -} - -// MigrationRecord is the per-scope legacy-home migration ledger entry. -type MigrationRecord struct { - ScopeHash string `json:"scope_hash"` - State MigrationState `json:"state"` - ImportedSessions []string `json:"imported_sessions,omitempty"` - ValidatedSession string `json:"validated_session,omitempty"` - UpdatedAt time.Time `json:"updated_at,omitempty"` -} - -// RuntimeState is the `.storage/runtime.json` document. -type RuntimeState struct { - SchemaVersion int `json:"schema_version"` - RootIdentity string `json:"root_identity"` - Homes map[string]RuntimeHomeRecord `json:"homes,omitempty"` - Scratch map[string]JobScratchRecord `json:"scratch,omitempty"` - Migrations map[string]MigrationRecord `json:"migrations,omitempty"` - UpdatedAt time.Time `json:"updated_at,omitempty"` -} - -// NewRuntimeState builds an empty runtime metadata state bound to the root. -func NewRuntimeState(rootIdentity string) RuntimeState { - return RuntimeState{ - SchemaVersion: RuntimeSchemaVersion, - RootIdentity: rootIdentity, - Homes: map[string]RuntimeHomeRecord{}, - Scratch: map[string]JobScratchRecord{}, - Migrations: map[string]MigrationRecord{}, - } -} - -// RuntimeStore is the locked, atomically persisted `.storage/runtime.json` -// metadata store. It mirrors the sidecar Store semantics: the runtime flock is -// a leaf-level lock held only for short load/mutate/save sections, missing or -// corrupt files open rebuilt, and foreign-root or newer-schema files open -// report-only. Corruption causes reuse the sidecar's CorruptSidecarError so -// callers handle one evidence-loss shape. -type RuntimeStore struct { - mu sync.Mutex - dir string - path string - lockPath string - root string - identity string - state RuntimeState - status SidecarStatus - closed bool - now func() time.Time - backupError error - loadCause *CorruptSidecarError -} - -// OpenRuntimeStore loads the runtime metadata for the canonical workspace -// root, creating the private `.storage` directory when needed. -func OpenRuntimeStore(workspaceRoot string) (*RuntimeStore, error) { - root := strings.TrimSpace(workspaceRoot) - if root == "" { - return nil, fmt.Errorf("workspace root is required") - } - canonical, err := Canonicalize(root) - if err != nil { - return nil, err - } - identity, err := RootIdentity(canonical) - if err != nil { - return nil, err - } - dir := filepath.Join(canonical, StorageDirName) - if err := os.MkdirAll(dir, 0o700); err != nil { - return nil, fmt.Errorf("prepare storage directory: %w", err) - } - store := &RuntimeStore{ - dir: dir, - path: filepath.Join(dir, runtimeFileName), - lockPath: filepath.Join(dir, runtimeLockName), - root: canonical, - identity: identity, - now: func() time.Time { return time.Now().UTC() }, - } - if err := store.load(); err != nil { - return nil, err - } - return store, nil -} - -// Status reports how much trust this process may place in the loaded state. -func (s *RuntimeStore) Status() SidecarStatus { return s.status } - -// RootIdentity returns the identity hash bound to this store. -func (s *RuntimeStore) RootIdentity() string { return s.identity } - -// Path returns the runtime metadata file path. -func (s *RuntimeStore) Path() string { return s.path } - -// BackupError reports a non-fatal failure to preserve a corrupt file. -func (s *RuntimeStore) BackupError() error { return s.backupError } - -// LoadCause reports why the state was rebuilt from corrupt bytes, if it was. -func (s *RuntimeStore) LoadCause() *CorruptSidecarError { return s.loadCause } - -// Reload re-reads the runtime metadata under its lock so a long-lived store -// observes writes made by other stores on the same root. -func (s *RuntimeStore) Reload() error { - s.mu.Lock() - defer s.mu.Unlock() - if s.closed { - return fmt.Errorf("runtime metadata store is closed") - } - unlock, err := s.lock() - if err != nil { - return err - } - defer unlock() - return s.load() -} - -// State returns a copy of the currently loaded runtime state. -func (s *RuntimeStore) State() RuntimeState { - s.mu.Lock() - defer s.mu.Unlock() - return cloneRuntimeState(s.state) -} - -func (s *RuntimeStore) load() error { - s.state = NewRuntimeState(s.identity) - s.loadCause = nil - s.backupError = nil - data, err := os.ReadFile(s.path) - switch { - case errors.Is(err, os.ErrNotExist): - s.status = SidecarRebuilt - return nil - case err != nil: - return fmt.Errorf("read runtime metadata: %w", err) - } - if len(bytes.TrimSpace(data)) == 0 { - return s.rebuildFromCorrupt(fmt.Errorf("empty runtime metadata")) - } - var decoded RuntimeState - dec := json.NewDecoder(bytes.NewReader(data)) - if err := dec.Decode(&decoded); err != nil { - return s.rebuildFromCorrupt(err) - } - var trailing any - if err := dec.Decode(&trailing); err != io.EOF { - return s.rebuildFromCorrupt(fmt.Errorf("unexpected trailing JSON")) - } - if decoded.SchemaVersion > RuntimeSchemaVersion { - s.status = SidecarReportOnly - s.state = normalizeRuntimeState(decoded) - return nil - } - if decoded.RootIdentity != "" && decoded.RootIdentity != s.identity { - s.status = SidecarReportOnly - s.state = normalizeRuntimeState(decoded) - return nil - } - decoded = normalizeRuntimeState(decoded) - for id, record := range decoded.Scratch { - if !record.CleanupState.Valid() { - return s.rebuildFromCorrupt(fmt.Errorf("scratch record %q has invalid cleanup state", id)) - } - } - for hash, record := range decoded.Migrations { - if !record.State.Valid() { - return s.rebuildFromCorrupt(fmt.Errorf("migration record %q has invalid state", hash)) - } - } - s.state = decoded - s.status = SidecarReady - return nil -} - -func (s *RuntimeStore) rebuildFromCorrupt(cause error) error { - s.state = NewRuntimeState(s.identity) - s.status = SidecarRebuilt - s.loadCause = &CorruptSidecarError{Path: s.path, Err: cause} - if _, err := s.backupCorruptCopy(); err != nil { - s.backupError = err - } - return nil -} - -func (s *RuntimeStore) backupCorruptCopy() (string, error) { - data, err := os.ReadFile(s.path) - if err != nil || len(bytes.TrimSpace(data)) == 0 { - return "", err - } - backupDir := filepath.Join(s.dir, backupDirName) - if err := os.MkdirAll(backupDir, 0o700); err != nil { - return "", err - } - // Stable name: a persistently corrupt file must not grow backups - // unboundedly across passes. - backup := filepath.Join(backupDir, runtimeCorruptBackup) - if err := state.WriteAtomic(backup, data); err != nil { - return "", err - } - return backup, nil -} - -// Update runs one locked read-modify-write cycle on the runtime metadata. It -// fails on report-only stores so a foreign or newer file is never rewritten. -func (s *RuntimeStore) Update(mutate func(*RuntimeState) error) error { - if mutate == nil { - return fmt.Errorf("runtime metadata update callback is required") - } - s.mu.Lock() - defer s.mu.Unlock() - if s.closed { - return fmt.Errorf("runtime metadata store is closed") - } - if s.status == SidecarReportOnly { - return ErrReportOnly - } - unlock, err := s.lock() - if err != nil { - return err - } - defer unlock() - // Re-load inside the lock so interleaved writers never lose updates. - if err := s.load(); err != nil { - return err - } - if s.status == SidecarReportOnly { - return ErrReportOnly - } - next := cloneRuntimeState(s.state) - if err := mutate(&next); err != nil { - return err - } - if next.SchemaVersion != RuntimeSchemaVersion { - return fmt.Errorf("runtime metadata schema version must stay %d", RuntimeSchemaVersion) - } - if next.RootIdentity != s.identity { - return fmt.Errorf("runtime metadata root identity must stay bound to this root") - } - next = normalizeRuntimeState(next) - next.UpdatedAt = s.now() - data, err := json.MarshalIndent(next, "", " ") - if err != nil { - return err - } - data = append(data, '\n') - if err := state.WriteAtomic(s.path, data); err != nil { - return err - } - s.state = next - s.status = SidecarReady - return nil -} - -func (s *RuntimeStore) lock() (func(), error) { - file, err := os.OpenFile(s.lockPath, os.O_RDWR|os.O_CREATE, 0o600) - if err != nil { - return nil, fmt.Errorf("open runtime metadata lock: %w", err) - } - if err := flockExclusive(file); err != nil { - _ = file.Close() - return nil, fmt.Errorf("lock runtime metadata: %w", err) - } - return func() { - _ = flockUnlock(file) - _ = file.Close() - }, nil -} - -// Close releases the store handle; the flock itself is never held between -// operations, so close only gates further use. -func (s *RuntimeStore) Close() error { - s.mu.Lock() - defer s.mu.Unlock() - s.closed = true - return nil -} - -func normalizeRuntimeState(st RuntimeState) RuntimeState { - if st.Homes == nil { - st.Homes = map[string]RuntimeHomeRecord{} - } - if st.Scratch == nil { - st.Scratch = map[string]JobScratchRecord{} - } - if st.Migrations == nil { - st.Migrations = map[string]MigrationRecord{} - } - return st -} - -func cloneRuntimeState(st RuntimeState) RuntimeState { - clone := st - clone.Homes = make(map[string]RuntimeHomeRecord, len(st.Homes)) - for hash, record := range st.Homes { - clone.Homes[hash] = record - } - clone.Scratch = make(map[string]JobScratchRecord, len(st.Scratch)) - for id, record := range st.Scratch { - clone.Scratch[id] = record - } - clone.Migrations = make(map[string]MigrationRecord, len(st.Migrations)) - for hash, record := range st.Migrations { - record.ImportedSessions = append([]string(nil), record.ImportedSessions...) - clone.Migrations[hash] = record - } - return clone -} diff --git a/internal/commentrunner/storage/runtime_store_test.go b/internal/commentrunner/storage/runtime_store_test.go deleted file mode 100644 index 05079023..00000000 --- a/internal/commentrunner/storage/runtime_store_test.go +++ /dev/null @@ -1,238 +0,0 @@ -package storage - -import ( - "errors" - "os" - "path/filepath" - "strings" - "testing" - "time" -) - -func TestOpenRuntimeStoreMissingRebuilds(t *testing.T) { - root := testRoot(t) - store, err := OpenRuntimeStore(root) - if err != nil { - t.Fatalf("OpenRuntimeStore: %v", err) - } - defer store.Close() - if store.Status() != SidecarRebuilt { - t.Fatalf("status = %q, want %q", store.Status(), SidecarRebuilt) - } - st := store.State() - if st.SchemaVersion != RuntimeSchemaVersion { - t.Fatalf("schema = %d, want %d", st.SchemaVersion, RuntimeSchemaVersion) - } - if st.RootIdentity != wantIdentity(t, root) { - t.Fatalf("root identity = %q, want %q", st.RootIdentity, wantIdentity(t, root)) - } - if len(st.Homes) != 0 || len(st.Scratch) != 0 || len(st.Migrations) != 0 { - t.Fatalf("fresh state must have empty maps: %+v", st) - } -} - -func TestRuntimeStoreUpdatePersistsAtomically(t *testing.T) { - root := testRoot(t) - store, err := OpenRuntimeStore(root) - if err != nil { - t.Fatalf("OpenRuntimeStore: %v", err) - } - home := RuntimeHomeRecord{ - Hash: strings.Repeat("ab", 16), Path: filepath.Join(root, RunnerHomesDirName, strings.Repeat("ab", 16)), - Hostname: "host-1", Repo: "o/r", Runner: "runner-1", CreatedAt: time.Now().UTC(), - } - if err := store.Update(func(st *RuntimeState) error { - st.Homes[home.Hash] = home - return nil - }); err != nil { - t.Fatalf("Update: %v", err) - } - if store.Status() != SidecarReady { - t.Fatalf("status after update = %q, want %q", store.Status(), SidecarReady) - } - if err := store.Close(); err != nil { - t.Fatalf("Close: %v", err) - } - reopened, err := OpenRuntimeStore(root) - if err != nil { - t.Fatalf("reopen: %v", err) - } - defer reopened.Close() - if reopened.Status() != SidecarReady { - t.Fatalf("reopened status = %q, want %q", reopened.Status(), SidecarReady) - } - got, ok := reopened.State().Homes[home.Hash] - if !ok || got != home { - t.Fatalf("home record = %+v ok=%v, want %+v", got, ok, home) - } -} - -func TestRuntimeStoreUpdatePinsSchemaAndRoot(t *testing.T) { - root := testRoot(t) - store, err := OpenRuntimeStore(root) - if err != nil { - t.Fatalf("OpenRuntimeStore: %v", err) - } - defer store.Close() - if err := store.Update(func(st *RuntimeState) error { - st.SchemaVersion = RuntimeSchemaVersion + 1 - return nil - }); err == nil { - t.Fatalf("schema bump must be rejected") - } - if err := store.Update(func(st *RuntimeState) error { - st.RootIdentity = strings.Repeat("0", 64) - return nil - }); err == nil { - t.Fatalf("root identity change must be rejected") - } -} - -func TestRuntimeStoreCorruptBackupAndRebuild(t *testing.T) { - root := testRoot(t) - store, err := OpenRuntimeStore(root) - if err != nil { - t.Fatalf("OpenRuntimeStore: %v", err) - } - if err := store.Update(func(st *RuntimeState) error { - st.Scratch["job-aaaaaaaaaaaaaaaa"] = JobScratchRecord{JobID: "job-aaaaaaaaaaaaaaaa", CleanupState: CleanupManaged} - return nil - }); err != nil { - t.Fatalf("Update: %v", err) - } - path := store.Path() - if err := store.Close(); err != nil { - t.Fatalf("Close: %v", err) - } - corrupt := []byte("{not json") - if err := os.WriteFile(path, corrupt, 0o600); err != nil { - t.Fatalf("corrupt: %v", err) - } - rebuilt, err := OpenRuntimeStore(root) - if err != nil { - t.Fatalf("reopen corrupt: %v", err) - } - defer rebuilt.Close() - if rebuilt.Status() != SidecarRebuilt { - t.Fatalf("status = %q, want %q", rebuilt.Status(), SidecarRebuilt) - } - if rebuilt.LoadCause() == nil { - t.Fatalf("load cause must explain the rebuild") - } - backup := filepath.Join(root, StorageDirName, backupDirName, runtimeCorruptBackup) - data, err := os.ReadFile(backup) - if err != nil { - t.Fatalf("corrupt backup missing: %v", err) - } - if string(data) != string(corrupt) { - t.Fatalf("corrupt backup content mismatch") - } - if len(rebuilt.State().Scratch) != 0 { - t.Fatalf("rebuilt state must drop corrupt records") - } -} - -func TestRuntimeStoreNewerSchemaReportOnly(t *testing.T) { - root := testRoot(t) - path := filepath.Join(root, StorageDirName, runtimeFileName) - payload := []byte(`{"schema_version":2,"root_identity":"` + wantIdentity(t, root) + `"}`) - if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil { - t.Fatalf("mkdir: %v", err) - } - if err := os.WriteFile(path, payload, 0o600); err != nil { - t.Fatalf("write: %v", err) - } - store, err := OpenRuntimeStore(root) - if err != nil { - t.Fatalf("OpenRuntimeStore: %v", err) - } - defer store.Close() - if store.Status() != SidecarReportOnly { - t.Fatalf("status = %q, want %q", store.Status(), SidecarReportOnly) - } - if store.State().SchemaVersion != 2 { - t.Fatalf("report-only state must expose the newer schema for inventory") - } - if err := store.Update(func(st *RuntimeState) error { return nil }); !errors.Is(err, ErrReportOnly) { - t.Fatalf("Update err = %v, want ErrReportOnly", err) - } -} - -func TestRuntimeStoreForeignRootReportOnly(t *testing.T) { - root := testRoot(t) - path := filepath.Join(root, StorageDirName, runtimeFileName) - payload := []byte(`{"schema_version":1,"root_identity":"` + strings.Repeat("0", 64) + `"}`) - if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil { - t.Fatalf("mkdir: %v", err) - } - if err := os.WriteFile(path, payload, 0o600); err != nil { - t.Fatalf("write: %v", err) - } - store, err := OpenRuntimeStore(root) - if err != nil { - t.Fatalf("OpenRuntimeStore: %v", err) - } - defer store.Close() - if store.Status() != SidecarReportOnly { - t.Fatalf("status = %q, want %q", store.Status(), SidecarReportOnly) - } - if err := store.Update(func(st *RuntimeState) error { return nil }); !errors.Is(err, ErrReportOnly) { - t.Fatalf("Update err = %v, want ErrReportOnly", err) - } -} - -func TestRuntimeStoreSequentialWritersPreserved(t *testing.T) { - root := testRoot(t) - first, err := OpenRuntimeStore(root) - if err != nil { - t.Fatalf("open first: %v", err) - } - defer first.Close() - second, err := OpenRuntimeStore(root) - if err != nil { - t.Fatalf("open second: %v", err) - } - defer second.Close() - if err := first.Update(func(st *RuntimeState) error { - st.Homes["home-1"] = RuntimeHomeRecord{Hash: "home-1"} - return nil - }); err != nil { - t.Fatalf("first update: %v", err) - } - if err := second.Update(func(st *RuntimeState) error { - st.Scratch["job-cccccccccccccccc"] = JobScratchRecord{JobID: "job-cccccccccccccccc", CleanupState: CleanupManaged} - return nil - }); err != nil { - t.Fatalf("second update: %v", err) - } - if err := first.Reload(); err != nil { - t.Fatalf("reload: %v", err) - } - st := first.State() - if _, ok := st.Homes["home-1"]; !ok { - t.Fatalf("first writer's home record lost: %+v", st.Homes) - } - if _, ok := st.Scratch["job-cccccccccccccccc"]; !ok { - t.Fatalf("second writer's scratch record lost: %+v", st.Scratch) - } -} - -func TestRuntimeStoreInvalidCleanupStateRebuilds(t *testing.T) { - root := testRoot(t) - path := filepath.Join(root, StorageDirName, runtimeFileName) - payload := []byte(`{"schema_version":1,"root_identity":"` + wantIdentity(t, root) + `","scratch":{"job-x":{"job_id":"job-x","cleanup_state":"bogus"}}}`) - if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil { - t.Fatalf("mkdir: %v", err) - } - if err := os.WriteFile(path, payload, 0o600); err != nil { - t.Fatalf("write: %v", err) - } - store, err := OpenRuntimeStore(root) - if err != nil { - t.Fatalf("OpenRuntimeStore: %v", err) - } - defer store.Close() - if store.Status() != SidecarRebuilt { - t.Fatalf("status = %q, want %q for invalid cleanup state", store.Status(), SidecarRebuilt) - } -} diff --git a/internal/commentrunner/storage/service.go b/internal/commentrunner/storage/service.go index 8626353d..d561296a 100644 --- a/internal/commentrunner/storage/service.go +++ b/internal/commentrunner/storage/service.go @@ -6,7 +6,6 @@ import ( "fmt" "os" "path/filepath" - "sort" "strings" "sync" "time" @@ -46,7 +45,6 @@ type Service struct { mu sync.Mutex root string store *Store - runtime *RuntimeStore stateLoader StateLoader poolInspector PoolInspector poolRemover PoolRemover @@ -123,32 +121,8 @@ func (s *Service) Root() string { return s.root } // Store exposes the sidecar for lifecycle tests and close management. func (s *Service) Store() *Store { return s.store } -// RuntimeStore returns the lazily opened `.storage/runtime.json` metadata -// store, shared by every entry point of this logical run. -func (s *Service) RuntimeStore() (*RuntimeStore, error) { - s.mu.Lock() - defer s.mu.Unlock() - if s.runtime != nil { - return s.runtime, nil - } - store, err := OpenRuntimeStore(s.root) - if err != nil { - return nil, err - } - s.runtime = store - return store, nil -} - -// Close releases the sidecar and the runtime metadata store. +// Close releases the sidecar. func (s *Service) Close() error { - s.mu.Lock() - runtime := s.runtime - s.mu.Unlock() - if runtime != nil { - if err := runtime.Close(); err != nil { - return err - } - } return s.store.Close() } @@ -180,97 +154,156 @@ func (s *Service) AdmitDispatch(ctx context.Context) error { s.mu.Lock() s.lastPressureReconcile = s.now() s.mu.Unlock() + // Under pressure the shared layout also reclaims stale job scratch and + // rebuildable home caches. Best-effort: failures are reported only when + // pressure persists, never fail admission by themselves. + var cleanupDiags []string + if s.hasRunnerHomeRecords() { + if _, err := s.ReconcileJobScratch(ctx, true); err != nil { + cleanupDiags = append(cleanupDiags, safeDiagnostic("pressured job scratch reconciliation: "+err.Error())) + } + if _, err := s.EvictRuntimeCaches(ctx, true); err != nil { + cleanupDiags = append(cleanupDiags, safeDiagnostic("pressured runtime cache eviction: "+err.Error())) + } + } free, err = s.statfs(s.root) if err != nil { return fmt.Errorf("storage admission statfs recheck: %w", err) } if free < uint64(s.minFreeBytes) { - return fmt.Errorf("%w: %d bytes free of %d required after cleanup", ErrStoragePressure, free, s.minFreeBytes) + msg := fmt.Sprintf("%d bytes free of %d required after cleanup", free, s.minFreeBytes) + for _, diag := range cleanupDiags { + msg += "; " + diag + } + return fmt.Errorf("%w: %s", ErrStoragePressure, msg) } return nil } +// hasRunnerHomeRecords reports whether the sidecar tracks at least one shared +// runtime home, gating pressured scratch/cache cleanup to roots on the shared +// layout. +func (s *Service) hasRunnerHomeRecords() bool { + for _, resource := range s.store.State().Resources { + if resource.Kind == ResourceKindRunnerHome { + return true + } + } + return false +} + // RecordSessionResources upserts the exact runtime and PROCESS pool physical // identities for a session before the runtime is exposed to sandbox execution. // It is fail-closed: an upsert failure means the runtime would be unmanaged. func (s *Service) RecordSessionResources(_ context.Context, repo, publicSessionID, workspacePath string) error { + records, err := s.sessionResourceRecords(repo, publicSessionID, workspacePath) + if err != nil { + return err + } + if err := s.upsertResourceRecords(records); err != nil { + return fmt.Errorf("record session storage resources: %w", err) + } + return nil +} + +// RecordSessionProcessPool upserts only the session's PROCESS pool identity. +// The runner-scoped shared layout has no per-session `.sessions/` +// runtime, so the session_runtime record must not be created; the pool record +// remains fail-closed before sandbox exposure. +func (s *Service) RecordSessionProcessPool(_ context.Context, repo, publicSessionID, workspacePath string) error { + records, err := s.sessionResourceRecords(repo, publicSessionID, workspacePath) + if err != nil { + return err + } + pool := records[:0] + for _, record := range records { + if record.Kind == ResourceKindSessionProcessPool { + pool = append(pool, record) + } + } + if err := s.upsertResourceRecords(pool); err != nil { + return fmt.Errorf("record session process pool: %w", err) + } + return nil +} + +// sessionResourceRecords builds the desired runtime and pool records for one +// session, preserving FirstObservedAt of records the sidecar already holds. +func (s *Service) sessionResourceRecords(repo, publicSessionID, workspacePath string) ([]PhysicalResource, error) { repo = strings.TrimSpace(repo) publicSessionID = strings.TrimSpace(publicSessionID) workspacePath = strings.TrimSpace(workspacePath) if repo == "" || publicSessionID == "" || workspacePath == "" { - return fmt.Errorf("repo, public session id, and workspace path are required for storage recording") + return nil, fmt.Errorf("repo, public session id, and workspace path are required for storage recording") } runtimeHash, err := SessionRuntimeHash(repo, publicSessionID, workspacePath) if err != nil { - return err + return nil, err } canonical, err := Canonicalize(workspacePath) if err != nil { - return fmt.Errorf("canonicalize session workspace for storage recording: %w", err) + return nil, fmt.Errorf("canonicalize session workspace for storage recording: %w", err) } poolHash, err := SessionProcessPoolHash(repo, publicSessionID, canonical) if err != nil { - return err + return nil, err } now := s.now().UTC() - // Skip the write entirely when both records already match: steady-state - // touches must not fsync the sidecar per running job per pass. - desired := func(existing PhysicalResource, id string, kind ResourceKind, hash, path string) PhysicalResource { - record := existing + current := s.store.State() + records := make([]PhysicalResource, 0, 2) + for _, spec := range []struct { + kind ResourceKind + hash string + path string + }{ + {ResourceKindSessionRuntime, runtimeHash, RuntimeRootForHash(s.root, runtimeHash)}, + {ResourceKindSessionProcessPool, poolHash, ProcessPoolRootForHash(s.root, poolHash)}, + } { + id := ResourceID(spec.kind, repo, publicSessionID, spec.hash) + record := current.Resources[id] if record.ID == "" { - record = PhysicalResource{ID: id, Kind: kind, FirstObservedAt: now} + record = PhysicalResource{ID: id, Kind: spec.kind, FirstObservedAt: now} } - record.Path = path + record.Path = spec.path record.Repo = repo record.PublicSessionID = publicSessionID - record.PhysicalHash = hash + record.PhysicalHash = spec.hash record.CleanupState = CleanupManaged record.CleanupAttemptID = "" record.LastCleanupError = "" - return record + records = append(records, record) } + return records, nil +} + +// upsertResourceRecords writes the desired records in one locked update, +// skipping the write entirely when every record already matches: steady-state +// touches must not fsync the sidecar per running job per pass. +func (s *Service) upsertResourceRecords(records []PhysicalResource) error { current := s.store.State() - runtimeID := ResourceID(ResourceKindSessionRuntime, repo, publicSessionID, runtimeHash) - poolID := ResourceID(ResourceKindSessionProcessPool, repo, publicSessionID, poolHash) - if current.Resources[runtimeID] == desired(current.Resources[runtimeID], runtimeID, ResourceKindSessionRuntime, runtimeHash, RuntimeRootForHash(s.root, runtimeHash)) && - current.Resources[poolID] == desired(current.Resources[poolID], poolID, ResourceKindSessionProcessPool, poolHash, ProcessPoolRootForHash(s.root, poolHash)) { + unchanged := true + for _, record := range records { + if current.Resources[record.ID] != record { + unchanged = false + break + } + } + if unchanged { return nil } - upsert := func(st *StorageState) error { - for _, spec := range []struct { - kind ResourceKind - hash string - path string - }{ - {ResourceKindSessionRuntime, runtimeHash, RuntimeRootForHash(s.root, runtimeHash)}, - {ResourceKindSessionProcessPool, poolHash, ProcessPoolRootForHash(s.root, poolHash)}, - } { - id := ResourceID(spec.kind, repo, publicSessionID, spec.hash) - record := st.Resources[id] - if record.ID == "" { - record = PhysicalResource{ID: id, Kind: spec.kind, FirstObservedAt: now} - } - record.Path = spec.path - record.Repo = repo - record.PublicSessionID = publicSessionID - record.PhysicalHash = spec.hash - record.CleanupState = CleanupManaged - record.CleanupAttemptID = "" - record.LastCleanupError = "" - st.Resources[id] = record + return s.store.Update(func(st *StorageState) error { + for _, record := range records { + st.Resources[record.ID] = record } return nil - } - if err := s.store.Update(upsert); err != nil { - return fmt.Errorf("record session storage resources: %w", err) - } - return nil + }) } // RecordRuntimeHome upserts the prepared runtime home of one scope before the // home is exposed to sandbox execution. It is fail-closed: the recorded root // must be the scope's canonical home below this root, and an upsert failure -// means the home would be unmanaged. +// means the home would be unmanaged. The record lives in the shared sidecar: +// one metadata authority owns every physical resource. func (s *Service) RecordRuntimeHome(_ context.Context, scope RuntimeScope, paths RuntimeHomePaths) error { if err := scope.Validate(); err != nil { return err @@ -283,32 +316,19 @@ func (s *Service) RecordRuntimeHome(_ context.Context, scope RuntimeScope, paths if filepath.Clean(paths.Root) != expected { return fmt.Errorf("runtime home path %q does not match the scoped home %q for this root", paths.Root, expected) } - store, err := s.RuntimeStore() - if err != nil { - return err - } now := s.now().UTC() - desired := func(existing RuntimeHomeRecord) RuntimeHomeRecord { - record := existing - if record.Hash == "" { - record = RuntimeHomeRecord{Hash: hash, CreatedAt: now} - } - record.Path = expected - record.Hostname = scope.Hostname - record.Realm = scope.Realm - record.Repo = scope.Repo - record.Runner = scope.Runner - return record - } - // Skip the write when the record already matches: steady-state touches - // must not fsync the metadata store per dispatched job. - if current := store.State().Homes[hash]; current == desired(current) { - return nil - } - if err := store.Update(func(st *RuntimeState) error { - st.Homes[hash] = desired(st.Homes[hash]) - return nil - }); err != nil { + id := ResourceID(ResourceKindRunnerHome, scope.Repo, "", hash) + record := s.store.State().Resources[id] + if record.ID == "" { + record = PhysicalResource{ID: id, Kind: ResourceKindRunnerHome, FirstObservedAt: now} + } + record.Path = expected + record.Repo = scope.Repo + record.PhysicalHash = hash + record.CleanupState = CleanupManaged + record.CleanupAttemptID = "" + record.LastCleanupError = "" + if err := s.upsertResourceRecords([]PhysicalResource{record}); err != nil { return fmt.Errorf("record runtime home: %w", err) } return nil @@ -316,7 +336,11 @@ func (s *Service) RecordRuntimeHome(_ context.Context, scope RuntimeScope, paths // RecordJobScratch upserts one job's scratch directory as managed. The path // must be the job's canonical scratch root below this root. -func (s *Service) RecordJobScratch(_ context.Context, jobID, path string) error { +func (s *Service) RecordJobScratch(_ context.Context, repo, jobID, path string) error { + repo = strings.TrimSpace(repo) + if repo == "" { + return fmt.Errorf("repo is required for job scratch recording") + } if !jobScratchIDPattern.MatchString(jobID) { return fmt.Errorf("job id %q is not a valid scratch identity", jobID) } @@ -324,69 +348,48 @@ func (s *Service) RecordJobScratch(_ context.Context, jobID, path string) error if filepath.Clean(path) != expected { return fmt.Errorf("job scratch path %q does not match the job scratch root %q for this root", path, expected) } - store, err := s.RuntimeStore() - if err != nil { - return err - } now := s.now().UTC() - desired := func(existing JobScratchRecord) JobScratchRecord { - record := existing - if record.JobID == "" { - record = JobScratchRecord{JobID: jobID, CreatedAt: now} - } - record.Path = expected - record.CleanupState = CleanupManaged - return record - } - if current := store.State().Scratch[jobID]; current == desired(current) { - return nil - } - if err := store.Update(func(st *RuntimeState) error { - st.Scratch[jobID] = desired(st.Scratch[jobID]) - return nil - }); err != nil { + id := ResourceID(ResourceKindJobScratch, repo, "", jobID) + record := s.store.State().Resources[id] + if record.ID == "" { + record = PhysicalResource{ID: id, Kind: ResourceKindJobScratch, FirstObservedAt: now} + } + record.Path = expected + record.Repo = repo + record.PhysicalHash = jobID + record.CleanupState = CleanupManaged + record.CleanupAttemptID = "" + record.LastCleanupError = "" + if err := s.upsertResourceRecords([]PhysicalResource{record}); err != nil { return fmt.Errorf("record job scratch: %w", err) } return nil } -// CompleteJobScratch removes one terminal job's scratch through the -// recoverable lifecycle: mark deleting, remove the directory capability -// scoped, then garbage-collect the record. A missing directory completes -// idempotently and an unknown job ID is a no-op. -func (s *Service) CompleteJobScratch(_ context.Context, jobID string) error { +// CompleteJobScratch removes one terminal job's scratch: the directory goes +// through capability-scoped deletion and its sidecar record is deleted. A +// missing record or directory completes idempotently; a crash between the two +// steps leaves a stale record the next engine sweep drops with a diagnostic. +func (s *Service) CompleteJobScratch(_ context.Context, repo, jobID string) error { + repo = strings.TrimSpace(repo) if !jobScratchIDPattern.MatchString(jobID) { return fmt.Errorf("job id %q is not a valid scratch identity", jobID) } - store, err := s.RuntimeStore() - if err != nil { - return err - } - record, ok := store.State().Scratch[jobID] + id := ResourceID(ResourceKindJobScratch, repo, "", jobID) + record, ok := s.store.State().Resources[id] if !ok { return nil } if err := validateScratchDeletionTarget(s.root, record.Path, jobID); err != nil { return fmt.Errorf("complete job scratch %q: %w", jobID, err) } - if err := store.Update(func(st *RuntimeState) error { - current, ok := st.Scratch[jobID] - if !ok { - return nil - } - current.CleanupState = CleanupDeleting - st.Scratch[jobID] = current - return nil - }); err != nil { - return fmt.Errorf("mark job scratch %q deleting: %w", jobID, err) - } if err := removeOpenedTree(record.Path, nil); err != nil { return fmt.Errorf("remove job scratch %q: %w", jobID, err) } // The directory is gone: drop the record instead of leaving a removed // tombstone, since job scratch has no orphan-observation lifecycle. - if err := store.Update(func(st *RuntimeState) error { - delete(st.Scratch, jobID) + if err := s.store.Update(func(st *StorageState) error { + delete(st.Resources, id) return nil }); err != nil { return fmt.Errorf("finalize job scratch %q: %w", jobID, err) @@ -394,69 +397,6 @@ func (s *Service) CompleteJobScratch(_ context.Context, jobID string) error { return nil } -// RuntimeMigrationLedger returns the migration ledger entry for one scope -// hash, if present. -func (s *Service) RuntimeMigrationLedger(_ context.Context, scopeHash string) (MigrationRecord, bool, error) { - if !ValidHashName(scopeHash) { - return MigrationRecord{}, false, fmt.Errorf("scope hash %q is not a complete scope hash", scopeHash) - } - store, err := s.RuntimeStore() - if err != nil { - return MigrationRecord{}, false, err - } - record, ok := store.State().Migrations[scopeHash] - return record, ok, nil -} - -// MarkRuntimeMigration upserts a migration ledger entry. Ledger states are -// monotonic (imported < validated < retired): regressions are rejected, and -// ImportedSessions merge as a union so concurrent resume validations never -// shrink the proof set. -func (s *Service) MarkRuntimeMigration(_ context.Context, record MigrationRecord) error { - if !ValidHashName(record.ScopeHash) { - return fmt.Errorf("scope hash %q is not a complete scope hash", record.ScopeHash) - } - if !record.State.Valid() { - return fmt.Errorf("migration state %q is invalid", record.State) - } - store, err := s.RuntimeStore() - if err != nil { - return err - } - if err := store.Update(func(st *RuntimeState) error { - existing, ok := st.Migrations[record.ScopeHash] - if ok && migrationStateRank(record.State) < migrationStateRank(existing.State) { - return fmt.Errorf("migration ledger for scope %q cannot regress from %q to %q", record.ScopeHash, existing.State, record.State) - } - merged := record - if ok { - merged.ImportedSessions = unionStrings(existing.ImportedSessions, record.ImportedSessions) - if merged.ValidatedSession == "" { - merged.ValidatedSession = existing.ValidatedSession - } - } - st.Migrations[record.ScopeHash] = merged - return nil - }); err != nil { - return fmt.Errorf("mark runtime migration: %w", err) - } - return nil -} - -func unionStrings(left, right []string) []string { - seen := map[string]bool{} - var merged []string - for _, value := range append(append([]string(nil), left...), right...) { - if value == "" || seen[value] { - continue - } - seen[value] = true - merged = append(merged, value) - } - sort.Strings(merged) - return merged -} - // ReconcileStorage runs the shared engine under the canonical root owner: // every destructive storage pass proves ownership first, so an admission // cleanup inside a standalone dispatch can never bypass the owner lock. diff --git a/internal/commentrunner/storage/storage.go b/internal/commentrunner/storage/storage.go index 90cb34de..305caa1b 100644 --- a/internal/commentrunner/storage/storage.go +++ b/internal/commentrunner/storage/storage.go @@ -49,10 +49,22 @@ type ResourceKind string const ( ResourceKindSessionRuntime ResourceKind = "session_runtime" ResourceKindSessionProcessPool ResourceKind = "session_process_pool" + // ResourceKindRunnerHome tracks one runner-scoped shared runtime HOME root. + // The record's repo segment is the scope repo, the session segment stays + // empty, and the hash segment is the scope hash. + ResourceKindRunnerHome ResourceKind = "runner_home" + // ResourceKindJobScratch tracks one job's disposable scratch root. The + // record's hash segment is the exact job ID. + ResourceKindJobScratch ResourceKind = "job_scratch" ) func (k ResourceKind) Valid() bool { - return k == ResourceKindSessionRuntime || k == ResourceKindSessionProcessPool + switch k { + case ResourceKindSessionRuntime, ResourceKindSessionProcessPool, ResourceKindRunnerHome, ResourceKindJobScratch: + return true + default: + return false + } } // CleanupState is the sidecar-persisted deletion lifecycle of one resource: From 5663ed5d99a5b908fd63cc33d9e6f88fd0efcfa7 Mon Sep 17 00:00:00 2001 From: johnlanni Date: Sat, 8 Aug 2026 22:26:17 +0800 Subject: [PATCH 06/17] feat(sandbox): bind per-job scratch dirs at fixed paths Add JobTmpDir/JobGoTmpDir/JobXDGDataHome/JobXDGStateHome to the sandbox Config. In bubblewrap mode each configured host dir is bound read-write at a fixed path below /tmp/issue-spec-scratch using the same --dir + --bind pattern as the temporary HOME mounts; in explicit unsafe mode the host paths are exported directly. TMPDIR, GOTMPDIR, XDG_DATA_HOME, and XDG_STATE_HOME join the protected environment set so command env cannot override them, and the writable-bind reservation rejects operator binds overlapping the job scratch dirs. Empty fields preserve the legacy behavior exactly. Signed-off-by: johnlanni --- internal/sandbox/bwrap_linux.go | 29 ++++++ internal/sandbox/bwrap_linux_test.go | 64 +++++++++++++ internal/sandbox/sandbox.go | 89 +++++++++++++++++- internal/sandbox/scratch_test.go | 134 +++++++++++++++++++++++++++ 4 files changed, 312 insertions(+), 4 deletions(-) create mode 100644 internal/sandbox/scratch_test.go diff --git a/internal/sandbox/bwrap_linux.go b/internal/sandbox/bwrap_linux.go index 0253ea9d..792513de 100644 --- a/internal/sandbox/bwrap_linux.go +++ b/internal/sandbox/bwrap_linux.go @@ -209,6 +209,35 @@ func buildBwrapCommand(cfg Config, target Command, env []string, bwrapPath strin if strings.TrimSpace(cfg.AcpxRuntimeDir) != "" { seenDirs[acpxSocketDir] = true } + // Per-job scratch: each configured host dir lands read-write at its fixed + // sandbox path. The scratch base is not covered by any existing mount, so + // create it and each destination explicitly before binding (the same + // --dir + --bind pattern as the temporary HOME mounts). + scratchDirs := []struct { + host string + dest string + }{ + {cfg.JobTmpDir, JobTmpSandboxPath}, + {cfg.JobGoTmpDir, JobGoTmpSandboxPath}, + {cfg.JobXDGDataHome, JobXDGDataSandboxPath}, + {cfg.JobXDGStateHome, JobXDGStateSandboxPath}, + } + scratchBaseCreated := false + for _, scratch := range scratchDirs { + host := filepath.Clean(strings.TrimSpace(scratch.host)) + if host == "." || !filepath.IsAbs(host) { + continue + } + if !scratchBaseCreated { + args = append(args, "--dir", JobScratchSandboxBase) + mounts = append(mounts, Mount{Destination: JobScratchSandboxBase, Mode: "dir"}) + seenDirs[JobScratchSandboxBase] = true + scratchBaseCreated = true + } + args = append(args, "--dir", scratch.dest, "--bind", host, scratch.dest) + mounts = append(mounts, Mount{Source: host, Destination: scratch.dest, Mode: "rw"}) + seenDirs[scratch.dest] = true + } if socket := strings.TrimSpace(cfg.HostSSHAgentSocket); socket != "" { socket = filepath.Clean(socket) args, mounts = appendBindParentDirs(args, mounts, HostSSHAgentSandboxPath, seenDirs, nil) diff --git a/internal/sandbox/bwrap_linux_test.go b/internal/sandbox/bwrap_linux_test.go index 65d8da53..09de6ade 100644 --- a/internal/sandbox/bwrap_linux_test.go +++ b/internal/sandbox/bwrap_linux_test.go @@ -356,6 +356,70 @@ func assertMount(t *testing.T, mounts []Mount, want Mount) { t.Fatalf("mounts missing %+v in %+v", want, mounts) } +func TestLinuxPrepareBindsJobScratchAtFixedPaths(t *testing.T) { + cfg := Config{ + BwrapPath: "/usr/bin/bwrap", + WorkspacePath: "/tmp/workspace", + TempHome: "/tmp/home", + TempGHConfigDir: "/tmp/gh", + TempXDGConfigHome: "/tmp/xdg", + TempCodexHome: "/tmp/codex", + JobTmpDir: "/tmp/job-scratch/tmp", + JobGoTmpDir: "/tmp/job-scratch/go-tmp", + JobXDGDataHome: "/tmp/job-scratch/xdg-data", + JobXDGStateHome: "/tmp/job-scratch/xdg-state", + HostEnv: []string{"PATH=/usr/bin"}, + SystemReadOnlyBinds: []string{"/usr"}, + } + prepared, err := Prepare(context.Background(), cfg, Command{Binary: "acpx", Args: []string{"run"}}, Dependencies{Runner: capableBwrapRunner(t)}) + if err != nil { + t.Fatalf("Prepare returned error: %v", err) + } + args := prepared.Command.Args + assertArgSequence(t, args, "--dir", JobScratchSandboxBase) + assertArgSequence(t, args, "--dir", JobTmpSandboxPath, "--bind", "/tmp/job-scratch/tmp", JobTmpSandboxPath) + assertArgSequence(t, args, "--dir", JobGoTmpSandboxPath, "--bind", "/tmp/job-scratch/go-tmp", JobGoTmpSandboxPath) + assertArgSequence(t, args, "--dir", JobXDGDataSandboxPath, "--bind", "/tmp/job-scratch/xdg-data", JobXDGDataSandboxPath) + assertArgSequence(t, args, "--dir", JobXDGStateSandboxPath, "--bind", "/tmp/job-scratch/xdg-state", JobXDGStateSandboxPath) + assertArgSequence(t, args, "--setenv", "TMPDIR", JobTmpSandboxPath) + assertArgSequence(t, args, "--setenv", "GOTMPDIR", JobGoTmpSandboxPath) + assertArgSequence(t, args, "--setenv", "XDG_DATA_HOME", JobXDGDataSandboxPath) + assertArgSequence(t, args, "--setenv", "XDG_STATE_HOME", JobXDGStateSandboxPath) + assertMount(t, prepared.Metadata.Mounts, Mount{Source: "/tmp/job-scratch/tmp", Destination: JobTmpSandboxPath, Mode: "rw"}) + assertMount(t, prepared.Metadata.Mounts, Mount{Source: "/tmp/job-scratch/go-tmp", Destination: JobGoTmpSandboxPath, Mode: "rw"}) + assertMount(t, prepared.Metadata.Mounts, Mount{Source: "/tmp/job-scratch/xdg-data", Destination: JobXDGDataSandboxPath, Mode: "rw"}) + assertMount(t, prepared.Metadata.Mounts, Mount{Source: "/tmp/job-scratch/xdg-state", Destination: JobXDGStateSandboxPath, Mode: "rw"}) + meta := prepared.Metadata.Env + if meta.TmpDir != JobTmpSandboxPath || meta.GoTmpDir != JobGoTmpSandboxPath || + meta.XDGDataHome != JobXDGDataSandboxPath || meta.XDGStateHome != JobXDGStateSandboxPath { + t.Fatalf("sandbox scratch metadata must use the fixed paths: %+v", meta) + } +} + +func TestLinuxPrepareWithoutJobScratchBindsNothing(t *testing.T) { + cfg := Config{ + BwrapPath: "/usr/bin/bwrap", + WorkspacePath: "/tmp/workspace", + TempHome: "/tmp/home", + TempGHConfigDir: "/tmp/gh", + TempXDGConfigHome: "/tmp/xdg", + HostEnv: []string{"PATH=/usr/bin"}, + SystemReadOnlyBinds: []string{"/usr"}, + } + prepared, err := Prepare(context.Background(), cfg, Command{Binary: "acpx"}, Dependencies{Runner: capableBwrapRunner(t)}) + if err != nil { + t.Fatalf("Prepare returned error: %v", err) + } + if argsContain(prepared.Command.Args, JobScratchSandboxBase) { + t.Fatalf("no scratch mount may appear without configured scratch dirs: %v", prepared.Command.Args) + } + for _, mount := range prepared.Metadata.Mounts { + if mount.Destination == JobScratchSandboxBase || strings.HasPrefix(mount.Destination, JobScratchSandboxBase+"/") { + t.Fatalf("unexpected scratch mount: %+v", mount) + } + } +} + func capableBwrapRunner(t *testing.T) Runner { t.Helper() return runnerFunc(func(ctx context.Context, command Command) (Result, error) { diff --git a/internal/sandbox/sandbox.go b/internal/sandbox/sandbox.go index 9648730f..b8ddf1d0 100644 --- a/internal/sandbox/sandbox.go +++ b/internal/sandbox/sandbox.go @@ -26,6 +26,14 @@ const ( HostSSHDirSandboxPath = "/tmp/issue-spec-home/.ssh" HostSSHAgentSandboxPath = "/run/issue-spec/ssh-agent.sock" + // JobScratchSandboxBase anchors the per-job disposable scratch mounts: + // TMPDIR, GOTMPDIR, XDG_DATA_HOME, and XDG_STATE_HOME of one job. + JobScratchSandboxBase = "/tmp/issue-spec-scratch" + JobTmpSandboxPath = JobScratchSandboxBase + "/tmp" + JobGoTmpSandboxPath = JobScratchSandboxBase + "/go-tmp" + JobXDGDataSandboxPath = JobScratchSandboxBase + "/xdg-data" + JobXDGStateSandboxPath = JobScratchSandboxBase + "/xdg-state" + defaultMinBwrapVersion = "0.5.0" ) @@ -82,7 +90,14 @@ type Config struct { TempXDGConfigHome string TempCodexHome string AcpxRuntimeDir string - HostGHConfigDir string + // JobTmpDir, JobGoTmpDir, JobXDGDataHome, and JobXDGStateHome are the + // job's disposable scratch directories on the host. Empty fields leave the + // corresponding environment untouched (legacy behavior). + JobTmpDir string + JobGoTmpDir string + JobXDGDataHome string + JobXDGStateHome string + HostGHConfigDir string // HostSSHDir and HostSSHAgentSocket are explicit opt-ins. In bubblewrap // mode the directory is mounted read-only at HOME/.ssh and the optional // Unix socket is mounted at a fixed path. In explicit unsafe mode the host @@ -204,6 +219,11 @@ type EnvMetadata struct { GHConfigDir string XDGConfigHome string CodexHome string + + TmpDir string + GoTmpDir string + XDGDataHome string + XDGStateHome string } type Mount struct { @@ -279,6 +299,10 @@ type envPaths struct { ghConfigDir string xdgConfigHome string codexHome string + tmpDir string + goTmpDir string + xdgDataHome string + xdgStateHome string } func hostEnvPaths(cfg Config) envPaths { @@ -286,11 +310,29 @@ func hostEnvPaths(cfg Config) envPaths { if cfg.UnsafeNoSandbox && strings.TrimSpace(cfg.HostSSHDir) != "" { home = filepath.Dir(filepath.Clean(cfg.HostSSHDir)) } - return envPaths{home: home, ghConfigDir: cfg.TempGHConfigDir, xdgConfigHome: cfg.TempXDGConfigHome, codexHome: cfg.TempCodexHome} + return envPaths{ + home: home, + ghConfigDir: cfg.TempGHConfigDir, + xdgConfigHome: cfg.TempXDGConfigHome, + codexHome: cfg.TempCodexHome, + tmpDir: cfg.JobTmpDir, + goTmpDir: cfg.JobGoTmpDir, + xdgDataHome: cfg.JobXDGDataHome, + xdgStateHome: cfg.JobXDGStateHome, + } } func sandboxEnvPaths() envPaths { - return envPaths{home: "/tmp/issue-spec-home", ghConfigDir: "/tmp/issue-spec-gh", xdgConfigHome: "/tmp/issue-spec-xdg", codexHome: "/tmp/issue-spec-codex"} + return envPaths{ + home: "/tmp/issue-spec-home", + ghConfigDir: "/tmp/issue-spec-gh", + xdgConfigHome: "/tmp/issue-spec-xdg", + codexHome: "/tmp/issue-spec-codex", + tmpDir: JobTmpSandboxPath, + goTmpDir: JobGoTmpSandboxPath, + xdgDataHome: JobXDGDataSandboxPath, + xdgStateHome: JobXDGStateSandboxPath, + } } type envBuildResult struct { @@ -318,6 +360,24 @@ func scrubEnvironment(cfg Config, paths envPaths, requireTempPaths bool) envBuil if strings.TrimSpace(cfg.TempCodexHome) != "" { codexHome = paths.codexHome } + // Job scratch env is opt-in per field: an unset host dir keeps the + // inherited/default behavior for that variable. + tmpDir := "" + if strings.TrimSpace(cfg.JobTmpDir) != "" { + tmpDir = paths.tmpDir + } + goTmpDir := "" + if strings.TrimSpace(cfg.JobGoTmpDir) != "" { + goTmpDir = paths.goTmpDir + } + xdgDataHome := "" + if strings.TrimSpace(cfg.JobXDGDataHome) != "" { + xdgDataHome = paths.xdgDataHome + } + xdgStateHome := "" + if strings.TrimSpace(cfg.JobXDGStateHome) != "" { + xdgStateHome = paths.xdgStateHome + } values := map[string]string{} meta := EnvMetadata{ @@ -325,6 +385,10 @@ func scrubEnvironment(cfg Config, paths envPaths, requireTempPaths bool) envBuil GHConfigDir: paths.ghConfigDir, XDGConfigHome: paths.xdgConfigHome, CodexHome: codexHome, + TmpDir: tmpDir, + GoTmpDir: goTmpDir, + XDGDataHome: xdgDataHome, + XDGStateHome: xdgStateHome, } for _, entry := range hostEnv { name, value, ok := strings.Cut(entry, "=") @@ -368,6 +432,18 @@ func scrubEnvironment(cfg Config, paths envPaths, requireTempPaths bool) envBuil if codexHome != "" { values["CODEX_HOME"] = codexHome } + if tmpDir != "" { + values["TMPDIR"] = tmpDir + } + if goTmpDir != "" { + values["GOTMPDIR"] = goTmpDir + } + if xdgDataHome != "" { + values["XDG_DATA_HOME"] = xdgDataHome + } + if xdgStateHome != "" { + values["XDG_STATE_HOME"] = xdgStateHome + } for _, capability := range cfg.FileCapabilities { values[capability.EnvName] = capability.Destination } @@ -430,7 +506,8 @@ func validatedWritableBinds(cfg Config) ([]string, error) { } workspace = filepath.Clean(canonicalWorkspace) } - reserved := []string{workspace, cfg.TempHome, cfg.TempGHConfigDir, cfg.TempXDGConfigHome, cfg.TempCodexHome, cfg.AcpxRuntimeDir, cfg.HostSSHDir, cfg.HostSSHAgentSocket} + reserved := []string{workspace, cfg.TempHome, cfg.TempGHConfigDir, cfg.TempXDGConfigHome, cfg.TempCodexHome, cfg.AcpxRuntimeDir, cfg.HostSSHDir, cfg.HostSSHAgentSocket, + cfg.JobTmpDir, cfg.JobGoTmpDir, cfg.JobXDGDataHome, cfg.JobXDGStateHome} reserved = append(reserved, cfg.ReadOnlyBinds...) systemBinds := cfg.SystemReadOnlyBinds if len(systemBinds) == 0 { @@ -554,6 +631,10 @@ func mergeCommandEnv(baseEntries, commandEntries []string, cfg Config, meta *Env "XDG_CONFIG_HOME": true, "CODEX_HOME": true, "SSH_AUTH_SOCK": true, + "TMPDIR": true, + "GOTMPDIR": true, + "XDG_DATA_HOME": true, + "XDG_STATE_HOME": true, } for _, entry := range commandEntries { name, value, ok := strings.Cut(entry, "=") diff --git a/internal/sandbox/scratch_test.go b/internal/sandbox/scratch_test.go new file mode 100644 index 00000000..db3de910 --- /dev/null +++ b/internal/sandbox/scratch_test.go @@ -0,0 +1,134 @@ +package sandbox + +import ( + "context" + "path/filepath" + "testing" +) + +// scratchConfig builds an unsafe-mode config with all four job scratch dirs +// below root. +func scratchConfig(root string) Config { + return Config{ + UnsafeNoSandbox: true, + WorkspacePath: filepath.Join(root, "workspace"), + TempHome: filepath.Join(root, "home"), + TempGHConfigDir: filepath.Join(root, "gh"), + TempXDGConfigHome: filepath.Join(root, "xdg"), + TempCodexHome: filepath.Join(root, "codex"), + JobTmpDir: filepath.Join(root, "scratch", "tmp"), + JobGoTmpDir: filepath.Join(root, "scratch", "go-tmp"), + JobXDGDataHome: filepath.Join(root, "scratch", "xdg-data"), + JobXDGStateHome: filepath.Join(root, "scratch", "xdg-state"), + HostEnv: []string{"PATH=/usr/bin", "TMPDIR=/host/tmp"}, + EnvAllowlist: []string{"PATH"}, + } +} + +func TestPrepareUnsafeSetsJobScratchEnv(t *testing.T) { + root := t.TempDir() + cfg := scratchConfig(root) + prepared, err := Prepare(context.Background(), cfg, Command{Binary: "acpx"}, Dependencies{}) + if err != nil { + t.Fatalf("Prepare returned error: %v", err) + } + env := envMap(prepared.Command.Env) + for name, want := range map[string]string{ + "TMPDIR": cfg.JobTmpDir, + "GOTMPDIR": cfg.JobGoTmpDir, + "XDG_DATA_HOME": cfg.JobXDGDataHome, + "XDG_STATE_HOME": cfg.JobXDGStateHome, + } { + if got := env[name]; got != want { + t.Fatalf("%s = %q, want %q in env %v", name, got, want, prepared.Command.Env) + } + } + meta := prepared.Metadata.Env + if meta.TmpDir != cfg.JobTmpDir || meta.GoTmpDir != cfg.JobGoTmpDir || + meta.XDGDataHome != cfg.JobXDGDataHome || meta.XDGStateHome != cfg.JobXDGStateHome { + t.Fatalf("scratch metadata missing: %+v", meta) + } +} + +func TestPrepareWithoutJobScratchLeavesEnvUntouched(t *testing.T) { + root := t.TempDir() + cfg := scratchConfig(root) + cfg.JobTmpDir, cfg.JobGoTmpDir, cfg.JobXDGDataHome, cfg.JobXDGStateHome = "", "", "", "" + prepared, err := Prepare(context.Background(), cfg, Command{Binary: "acpx"}, Dependencies{}) + if err != nil { + t.Fatalf("Prepare returned error: %v", err) + } + env := envMap(prepared.Command.Env) + for _, name := range []string{"TMPDIR", "GOTMPDIR", "XDG_DATA_HOME", "XDG_STATE_HOME"} { + if _, ok := env[name]; ok { + t.Fatalf("%s must stay unset without a job scratch dir: %v", name, prepared.Command.Env) + } + } + meta := prepared.Metadata.Env + if meta.TmpDir != "" || meta.GoTmpDir != "" || meta.XDGDataHome != "" || meta.XDGStateHome != "" { + t.Fatalf("scratch metadata must stay empty: %+v", meta) + } +} + +func TestPrepareJobScratchEnvSurvivesCommandEnvOverride(t *testing.T) { + root := t.TempDir() + cfg := scratchConfig(root) + commandEnv := []string{ + "TMPDIR=/evil/tmp", + "GOTMPDIR=/evil/go-tmp", + "XDG_DATA_HOME=/evil/data", + "XDG_STATE_HOME=/evil/state", + } + prepared, err := Prepare(context.Background(), cfg, Command{Binary: "acpx", Env: commandEnv}, Dependencies{}) + if err != nil { + t.Fatalf("Prepare returned error: %v", err) + } + env := envMap(prepared.Command.Env) + for name, want := range map[string]string{ + "TMPDIR": cfg.JobTmpDir, + "GOTMPDIR": cfg.JobGoTmpDir, + "XDG_DATA_HOME": cfg.JobXDGDataHome, + "XDG_STATE_HOME": cfg.JobXDGStateHome, + } { + if got := env[name]; got != want { + t.Fatalf("command env overrode protected %s: got %q, want %q", name, got, want) + } + } +} + +func TestWritableBindsRejectJobScratchOverlap(t *testing.T) { + root := t.TempDir() + workspace := filepath.Join(root, "workspace") + cfg := scratchConfig(root) + cfg.UnsafeNoSandbox = false + cfg.WorkspacePath = workspace + for _, dir := range []string{workspace, cfg.JobTmpDir, cfg.JobGoTmpDir, cfg.JobXDGDataHome, cfg.JobXDGStateHome} { + if err := mkdirPrivate(dir); err != nil { + t.Fatal(err) + } + } + for _, name := range []string{"tmp", "go-tmp", "xdg-data", "xdg-state"} { + bind := filepath.Join(root, "scratch", name) + if _, err := validatedWritableBinds(Config{ + WorkspacePath: workspace, + JobTmpDir: cfg.JobTmpDir, + JobGoTmpDir: cfg.JobGoTmpDir, + JobXDGDataHome: cfg.JobXDGDataHome, + JobXDGStateHome: cfg.JobXDGStateHome, + WritableBinds: []string{bind}, + }); err == nil { + t.Fatalf("writable bind overlapping job scratch %s must be rejected", name) + } + } + // The scratch parent overlaps every scratch dir and must also be rejected. + if _, err := validatedWritableBinds(Config{ + WorkspacePath: workspace, + JobTmpDir: cfg.JobTmpDir, + JobGoTmpDir: cfg.JobGoTmpDir, + JobXDGDataHome: cfg.JobXDGDataHome, + JobXDGStateHome: cfg.JobXDGStateHome, + WritableBinds: []string{filepath.Join(root, "scratch")}, + }); err == nil { + t.Fatalf("writable bind covering the job scratch root must be rejected") + } +} From 371bd200e0f3bf58be8da1fea8ce0146edebeee5 Mon Sep 17 00:00:00 2001 From: johnlanni Date: Sat, 8 Aug 2026 22:26:31 +0800 Subject: [PATCH 07/17] feat(runner): dispatch jobs on the runner-scoped shared HOME Wire the dispatcher to the runner-scoped shared runtime layout. A RuntimeIdentity (hostname, profile realm, runner identity) derived from the runner configuration pins every job of a repo to one shared runtime HOME below .runner-home/ and one disposable scratch tree below .job-scratch/; the zero identity keeps the legacy per-session layout for existing wiring. Recording stays fail-closed in order: runtime home, session process pool, then job scratch, all before sandbox exposure; terminal completion, failure, cancellation, and restart-recovered terminal transitions remove the job scratch best-effort with a bounded diagnostic on failure. Restart reconcile reuses the shared HOME without preparing new scratch. Shared-home mirror writes go through a single atomic helper: an identical existing file is left untouched (mode enforced) and every real write is temp-file plus rename, so concurrent jobs sharing one runtime home never observe partial gh/codex/profile mirrors and steady-state refreshes do not churn the shared home. The child-profile branch wipes the shared GH config dir only when it is non-empty, since steady-state child dispatches already see it empty and wiping would race concurrent jobs. Signed-off-by: johnlanni --- internal/commands/runner.go | 27 +- .../runner_repository_binding_test.go | 10 +- internal/commands/runner_serve_runtime.go | 5 + internal/commands/runner_storage.go | 12 + internal/commentrunner/jobs/cancel.go | 2 + internal/commentrunner/jobs/dispatcher.go | 356 ++++++++++++++---- internal/commentrunner/jobs/reconcile.go | 47 ++- .../jobs/storage_integration_test.go | 44 +++ 8 files changed, 421 insertions(+), 82 deletions(-) diff --git a/internal/commands/runner.go b/internal/commands/runner.go index dbf2a107..fe40433a 100644 --- a/internal/commands/runner.go +++ b/internal/commands/runner.go @@ -1110,6 +1110,10 @@ func (a *app) runRunnerReconcileWithStore(ctx context.Context, cfg commentrunner } workspaces := runnerWorkspaceManager(cfg) writebacks := wrapRunnerWriteback(&writeback.Service{GitHub: runnerBackend, Store: store}, a.runnerDiagnostics) + runtimeIdentity, err := jobs.RuntimeIdentityFor(cfg.Hostname, cfg.Profile, cfg.RunnerIdentity) + if err != nil { + return jobs.ReconcileResult{}, fmt.Errorf("runner runtime identity: %w", err) + } dispatcher := jobs.Dispatcher{ Store: store, Storage: runnerStorageLifecycle(ctx, cfg, store), @@ -1119,9 +1123,10 @@ func (a *app) runRunnerReconcileWithStore(ctx context.Context, cfg commentrunner BwrapPath: cfg.BwrapPath, HostGHConfigDir: cfg.GHConfigDir, }}, - Acpx: jobs.AcpxAdapterFactory{Config: jobs.NewAcpxConfig(cfg), RunnerConfig: cfg}, - Writeback: writebacks, - AcpxBinary: cfg.AcpxPath, + Acpx: jobs.AcpxAdapterFactory{Config: jobs.NewAcpxConfig(cfg), RunnerConfig: cfg}, + Writeback: writebacks, + AcpxBinary: cfg.AcpxPath, + RuntimeIdentity: runtimeIdentity, } return dispatcher.Reconcile(ctx) } @@ -1141,10 +1146,15 @@ func (a *app) runRunnerWorkspaceCleanupWithStore(ctx context.Context, cfg commen defer opened.Close() store = opened } + runtimeIdentity, err := jobs.RuntimeIdentityFor(cfg.Hostname, cfg.Profile, cfg.RunnerIdentity) + if err != nil { + return jobs.ReconcileResult{}, fmt.Errorf("runner runtime identity: %w", err) + } dispatcher := jobs.Dispatcher{ - Store: store, - Storage: runnerStorageLifecycle(ctx, cfg, store), - Workspaces: runnerWorkspaceManager(cfg), + Store: store, + Storage: runnerStorageLifecycle(ctx, cfg, store), + Workspaces: runnerWorkspaceManager(cfg), + RuntimeIdentity: runtimeIdentity, } return dispatcher.CleanupWorkspaces(ctx) } @@ -1235,6 +1245,10 @@ func (a *app) buildRunnerDispatcher(ctx context.Context, cfg commentrunner.Confi } workspaces := runnerWorkspaceManager(cfg) writebacks := wrapRunnerWriteback(&writeback.Service{GitHub: runnerBackend, Store: store}, a.runnerDiagnostics) + runtimeIdentity, err := jobs.RuntimeIdentityFor(cfg.Hostname, cfg.Profile, cfg.RunnerIdentity) + if err != nil { + return nil, nil, fmt.Errorf("runner runtime identity: %w", err) + } dispatcher := &jobs.Dispatcher{ Store: store, Storage: runnerStorageLifecycle(ctx, cfg, store), @@ -1250,6 +1264,7 @@ func (a *app) buildRunnerDispatcher(ctx context.Context, cfg commentrunner.Confi Writeback: writebacks, AcpxBinary: cfg.AcpxPath, IssueSpecBinary: issueSpecBinaryForRunner(), + RuntimeIdentity: runtimeIdentity, } return dispatcher, cleanup, nil } diff --git a/internal/commands/runner_repository_binding_test.go b/internal/commands/runner_repository_binding_test.go index 4a258ece..c27df743 100644 --- a/internal/commands/runner_repository_binding_test.go +++ b/internal/commands/runner_repository_binding_test.go @@ -5,6 +5,7 @@ import ( "context" "encoding/json" "errors" + "os" "path/filepath" "strings" "testing" @@ -86,7 +87,10 @@ func TestBuildRunnerDispatcherPinsAuthenticatedGitHubMetadataBeforeWorkspace(t * t.Fatal("caller-provided state store unexpectedly produced cleanup") } - workspaceRoot := t.TempDir() + workspaceRoot := filepath.Join(cfg.WorkspaceRoot, "ws-github-binding") + if err := os.MkdirAll(workspaceRoot, 0o700); err != nil { + t.Fatal(err) + } binding := testkit.WorkspaceBinding("ws-github-binding") binding.Workspace.Path = workspaceRoot binding.AcpxWorkingDirectory = workspaceRoot @@ -103,7 +107,7 @@ func TestBuildRunnerDispatcherPinsAuthenticatedGitHubMetadataBeforeWorkspace(t * dispatcher.TurnCorrelationID = func() (string, error) { return "turn-github-binding", nil } if err := store.Update(t.Context(), func(st *state.RunnerState) error { - _, _, err := st.CreateCommandJob(state.Job{ID: "job-github-binding", Repo: "o/r", IssueNumber: 379, + _, _, err := st.CreateCommandJob(state.Job{ID: "job-3793793793790001", Repo: "o/r", IssueNumber: 379, CommandID: "cmd-github-binding", CommandName: "new", CommandPrompt: "implement", CommandIdempotencyKey: "github-binding", SessionCreatorLogin: "alice", TriggeringUserLogin: "alice", TriggerCommentID: 37901, Status: state.StatusQueued, CreatedAt: testkit.Now, FirstObservedComment: state.SeenComment{ @@ -127,7 +131,7 @@ func TestBuildRunnerDispatcherPinsAuthenticatedGitHubMetadataBeforeWorkspace(t * t.Fatal(err) } pinned := workspaces.PrepareNewRequests[0].RepositoryBinding - job := store.Snapshot().Jobs["job-github-binding"] + job := store.Snapshot().Jobs["job-3793793793790001"] if !pinned.Equal(want.Binding) || !job.RepositoryBinding.Equal(want.Binding) || !job.DispatchIntent.RepositoryBinding.Equal(want.Binding) { t.Fatalf("authenticated metadata was not pinned before workspace: request=%+v job=%+v intent=%+v want=%+v", pinned, job.RepositoryBinding, job.DispatchIntent.RepositoryBinding, want.Binding) diff --git a/internal/commands/runner_serve_runtime.go b/internal/commands/runner_serve_runtime.go index ba45bd18..811507da 100644 --- a/internal/commands/runner_serve_runtime.go +++ b/internal/commands/runner_serve_runtime.go @@ -144,12 +144,17 @@ func defaultBuildRunnerServeRuntime(ctx context.Context, input runnerServeRuntim // Wrap the selected writeback boundary, including hermetic test or operator // dependencies, so every live lifecycle transition reaches diagnostics. writebacks = wrapRunnerWriteback(writebacks, input.Diagnostics) + runtimeIdentity, err := jobs.RuntimeIdentityFromProfile(input.Runner.Hostname, profile, input.Runner.RunnerIdentity) + if err != nil { + return nil, fmt.Errorf("runner runtime identity: %w", err) + } dispatcher := &jobs.Dispatcher{Store: input.Store, Storage: input.Storage, Repositories: repository.NativeResolver{Bindings: native, Scopes: scopes.ByRepository}, Workspaces: workspaces, Sandbox: sandboxer, Acpx: acpxFactory, Artifacts: artifacts, Writeback: writebacks, AcpxBinary: input.Runner.AcpxPath, IssueSpecBinary: issueSpecBinary, CredentialBroker: broker, CredentialScopes: scopes.ByRepository, CapabilityPreflight: broker, CapabilityHost: profile.Hostname, OperatorSkillDirs: input.Runner.OperatorSkillDirs, + RuntimeIdentity: runtimeIdentity, RequiredOperations: []capability.Operation{capability.OperationIssueRead, capability.OperationIssueCommentWrite, capability.OperationArtifactWrite, capability.OperationGitClone, capability.OperationGitPush}, } diff --git a/internal/commands/runner_storage.go b/internal/commands/runner_storage.go index 0319273a..17f08985 100644 --- a/internal/commands/runner_storage.go +++ b/internal/commands/runner_storage.go @@ -63,6 +63,18 @@ func (f failingStorageLifecycle) AdmitDispatch(context.Context) error { return f func (f failingStorageLifecycle) RecordSessionResources(context.Context, string, string, string) error { return f.err } +func (f failingStorageLifecycle) RecordSessionProcessPool(context.Context, string, string, string) error { + return f.err +} +func (f failingStorageLifecycle) RecordRuntimeHome(context.Context, storage.RuntimeScope, storage.RuntimeHomePaths) error { + return f.err +} +func (f failingStorageLifecycle) RecordJobScratch(context.Context, string, string, string) error { + return f.err +} +func (f failingStorageLifecycle) CompleteJobScratch(context.Context, string, string) error { + return f.err +} func (f failingStorageLifecycle) ReconcileStorage(context.Context, bool, bool) (storage.Report, error) { return storage.Report{}, f.err } diff --git a/internal/commentrunner/jobs/cancel.go b/internal/commentrunner/jobs/cancel.go index cbc597c6..86c66d1b 100644 --- a/internal/commentrunner/jobs/cancel.go +++ b/internal/commentrunner/jobs/cancel.go @@ -246,6 +246,7 @@ func (d *Dispatcher) cancelQueuedJob(ctx context.Context, cancel state.Cancellat }); err != nil { return err } + d.completeJobScratch(ctx, cancelled.Repo, cancelled.ID) _, err := d.Writeback.Write(ctx, writeback.Request{ Job: cancelled, Status: state.StatusCancelled, @@ -288,6 +289,7 @@ func (d *Dispatcher) cancelConfirmed(ctx context.Context, cancel state.Cancellat }); err != nil { return err } + d.completeJobScratch(ctx, cancelled.Repo, cancelled.ID) d.releaseLock(ctx, cancelled.ID, lock) _, err := d.Writeback.Write(ctx, writeback.Request{ Job: cancelled, diff --git a/internal/commentrunner/jobs/dispatcher.go b/internal/commentrunner/jobs/dispatcher.go index bff14d30..99673b30 100644 --- a/internal/commentrunner/jobs/dispatcher.go +++ b/internal/commentrunner/jobs/dispatcher.go @@ -1,8 +1,10 @@ package jobs import ( + "bytes" "context" "crypto/rand" + "crypto/sha256" "encoding/hex" "encoding/json" "errors" @@ -73,6 +75,12 @@ type SandboxRequest struct { RuntimeXDGConfigHome string RuntimeCodexHome string RuntimeAcpxDir string + // JobTmpDir/JobGoTmpDir/JobXDGDataHome/JobXDGStateHome are the job's + // disposable scratch dirs; set only on the runner-scoped shared layout. + JobTmpDir string + JobGoTmpDir string + JobXDGDataHome string + JobXDGStateHome string OperatorSkillDirs []string FileCapabilities []sandbox.FileCapability ChildProfile *clientauth.Profile @@ -131,12 +139,78 @@ type CapabilityPreflight interface { type StorageLifecycle interface { AdmitDispatch(ctx context.Context) error RecordSessionResources(ctx context.Context, repo, publicSessionID, workspacePath string) error + // RecordSessionProcessPool records only the session PROCESS pool; the + // shared layout has no per-session .sessions/ runtime to record. + RecordSessionProcessPool(ctx context.Context, repo, publicSessionID, workspacePath string) error + RecordRuntimeHome(ctx context.Context, scope storage.RuntimeScope, paths storage.RuntimeHomePaths) error + RecordJobScratch(ctx context.Context, repo, jobID, path string) error + CompleteJobScratch(ctx context.Context, repo, jobID string) error ReconcileStorage(ctx context.Context, apply, measureAll bool) (storage.Report, error) } +// RuntimeIdentity pins the dispatcher to one runner scope: every job of a +// repo shares the runner-scoped runtime HOME and receives its own disposable +// scratch. The zero value keeps the legacy per-session runtime layout so +// existing unit tests and non-runner wiring stay untouched. +type RuntimeIdentity struct { + Hostname string + Realm string + Runner string +} + +func (r RuntimeIdentity) enabled() bool { + return strings.TrimSpace(r.Hostname) != "" && strings.TrimSpace(r.Runner) != "" +} + +// scope builds the storage runtime scope for one repo under this identity. +func (r RuntimeIdentity) scope(repo string) storage.RuntimeScope { + return storage.RuntimeScope{ + Hostname: strings.TrimSpace(r.Hostname), + Realm: strings.TrimSpace(r.Realm), + Repo: strings.TrimSpace(repo), + Runner: strings.TrimSpace(r.Runner), + } +} + +// RuntimeIdentityFor derives the dispatcher runtime identity from the runner +// configuration: the named profile is resolved exactly as the runner scope +// path derivation resolves it, then RuntimeIdentityFromProfile computes the +// identity. +func RuntimeIdentityFor(hostname, profileName, runner string) (RuntimeIdentity, error) { + profile, _, err := clientauth.ResolveProfile(profileName, hostname) + if err != nil { + return RuntimeIdentity{}, err + } + return RuntimeIdentityFromProfile(hostname, profile, runner) +} + +// RuntimeIdentityFromProfile derives the dispatcher runtime identity from an +// already resolved profile: the normalized lowercase hostname, the profile +// realm (empty for the builtin GitHub profile; otherwise the lowercase +// profile name plus the realm-key digest exactly as the runner scope path +// derivation computes it), and the lowercase runner identity. The identity +// feeds only the scope hash preimage; it is never used as a path segment. +func RuntimeIdentityFromProfile(hostname string, profile clientauth.Profile, runner string) (RuntimeIdentity, error) { + host := strings.ToLower(clientauth.NormalizeHost(hostname)) + if host == "" { + return RuntimeIdentity{}, fmt.Errorf("hostname is required for the runner runtime identity") + } + realm := "" + if !clientauth.IsBuiltinGitHubProfile(profile) { + digest := sha256.Sum256([]byte(profile.RealmKey())) + realm = strings.ToLower(profile.Name) + "-" + hex.EncodeToString(digest[:8]) + } + runner = strings.ToLower(strings.TrimSpace(runner)) + if runner == "" { + return RuntimeIdentity{}, fmt.Errorf("runner identity is required for the runner runtime identity") + } + return RuntimeIdentity{Hostname: host, Realm: realm, Runner: runner}, nil +} + type Dispatcher struct { Store Store Storage StorageLifecycle + RuntimeIdentity RuntimeIdentity Repositories RepositoryResolver Workspaces WorkspaceManager Sandbox SandboxPreparer @@ -890,19 +964,19 @@ func (d *Dispatcher) prepareExecution(ctx context.Context, job state.Job, comman return ExecutionEnvironment{}, runnercontext.Bundle{}, "", err } integrationRoot := firstNonEmpty(execBinding.AcpxWorkingDirectory, execBinding.Workspace.Path, execBinding.SandboxWorkspacePath) - runtimePaths, err := stableSessionRuntimePaths(integrationRoot, job.Repo, publicID) + processRoot, err := prepareSessionProcessWorkspaceRoot(integrationRoot, job.Repo, publicID) if err != nil { return ExecutionEnvironment{}, runnercontext.Bundle{}, "", err } - processRoot, err := prepareSessionProcessWorkspaceRoot(integrationRoot, job.Repo, publicID) + layout, err := d.prepareRuntimeLayout(integrationRoot, job.Repo, publicID, job.ID) if err != nil { return ExecutionEnvironment{}, runnercontext.Bundle{}, "", err } - // Fail-closed: the exact runtime/pool identity is recorded before the - // runtime is exposed to sandbox execution so unmanaged runtimes never - // proliferate. + // Fail-closed: the exact runtime/pool/scratch identities are recorded + // before the runtime is exposed to sandbox execution so unmanaged + // runtimes never proliferate. if d.Storage != nil { - if err := d.Storage.RecordSessionResources(ctx, job.Repo, publicID, integrationRoot); err != nil { + if err := d.recordRuntimeLayout(ctx, job, publicID, integrationRoot, layout); err != nil { return ExecutionEnvironment{}, runnercontext.Bundle{}, "", fmt.Errorf("storage recording: %w", err) } } @@ -913,11 +987,15 @@ func (d *Dispatcher) prepareExecution(ctx context.Context, job state.Job, comman AcpxBinary: firstNonEmpty(d.AcpxBinary, acpx.DefaultBinary), IssueSpecBinary: d.IssueSpecBinary, ExtraEnv: extraEnv, - RuntimeHome: runtimePaths.home, - RuntimeGHConfigDir: runtimePaths.ghConfigDir, - RuntimeXDGConfigHome: runtimePaths.xdgConfigHome, - RuntimeCodexHome: runtimePaths.codexHome, - RuntimeAcpxDir: runtimePaths.acpxRuntimeDir, + RuntimeHome: layout.paths.home, + RuntimeGHConfigDir: layout.paths.ghConfigDir, + RuntimeXDGConfigHome: layout.paths.xdgConfigHome, + RuntimeCodexHome: layout.paths.codexHome, + RuntimeAcpxDir: layout.paths.acpxRuntimeDir, + JobTmpDir: layout.scratch.Tmp, + JobGoTmpDir: layout.scratch.GoTmp, + JobXDGDataHome: layout.scratch.XDGData, + JobXDGStateHome: layout.scratch.XDGState, OperatorSkillDirs: append([]string(nil), d.OperatorSkillDirs...), AcpxAgent: job.CoordinatorKind, ProcessWorkspaceRoot: processRoot, @@ -944,6 +1022,100 @@ func (d *Dispatcher) prepareExecution(ctx context.Context, job state.Job, comman return env, bundle, prompt, nil } +// runtimeLayout is the resolved runtime HOME and scratch placement for one +// dispatch: the runner-scoped shared home plus per-job scratch when a runtime +// identity is configured, the legacy per-session runtime otherwise. +type runtimeLayout struct { + paths sessionRuntimePaths + home storage.RuntimeHomePaths + scratch storage.JobScratchPaths + shared bool +} + +// prepareRuntimeLayout prepares the runtime HOME (and per-job scratch on the +// shared layout) fail-closed before any sandbox exposure. +func (d *Dispatcher) prepareRuntimeLayout(integrationRoot, repo, publicID, jobID string) (runtimeLayout, error) { + if !d.RuntimeIdentity.enabled() { + paths, err := stableSessionRuntimePaths(integrationRoot, repo, publicID) + if err != nil { + return runtimeLayout{}, err + } + return runtimeLayout{paths: paths}, nil + } + // The shared layout anchors .runner-home and .job-scratch at the same base + // the legacy per-session runtime root derives from the integration root. + workspaceRoot, err := runnerWorkspaceRootFor(integrationRoot) + if err != nil { + return runtimeLayout{}, err + } + scope := d.RuntimeIdentity.scope(repo) + home, err := storage.PrepareRuntimeHome(workspaceRoot, scope) + if err != nil { + return runtimeLayout{}, err + } + scratch, err := storage.PrepareJobScratch(workspaceRoot, jobID) + if err != nil { + return runtimeLayout{}, err + } + return runtimeLayout{ + paths: sessionRuntimePaths{ + home: home.Home, + ghConfigDir: home.GHConfigDir, + xdgConfigHome: home.XDGConfigHome, + codexHome: home.CodexHome, + acpxRuntimeDir: home.AcpxRuntimeDir, + }, + home: home, + scratch: scratch, + shared: true, + }, nil +} + +// recordRuntimeLayout persists the exact physical identities of the prepared +// layout: the shared home, session PROCESS pool, and job scratch on the +// shared layout; the per-session runtime and pool on the legacy layout. +func (d *Dispatcher) recordRuntimeLayout(ctx context.Context, job state.Job, publicID, integrationRoot string, layout runtimeLayout) error { + if !layout.shared { + return d.Storage.RecordSessionResources(ctx, job.Repo, publicID, integrationRoot) + } + if err := d.Storage.RecordRuntimeHome(ctx, d.RuntimeIdentity.scope(job.Repo), layout.home); err != nil { + return err + } + if err := d.Storage.RecordSessionProcessPool(ctx, job.Repo, publicID, integrationRoot); err != nil { + return err + } + return d.Storage.RecordJobScratch(ctx, job.Repo, job.ID, layout.scratch.Root) +} + +// runnerWorkspaceRootFor resolves the runner workspace root the shared layout +// anchors to: the parent of the session clone, exactly the base +// SessionRuntimeRoot derives from the same path. +func runnerWorkspaceRootFor(workspacePath string) (string, error) { + abs, err := filepath.Abs(strings.TrimSpace(workspacePath)) + if err != nil { + return "", fmt.Errorf("resolve workspace path for the runner root: %w", err) + } + clean := filepath.Clean(abs) + root := filepath.Dir(clean) + if root == clean { + return "", fmt.Errorf("workspace path %q cannot be filesystem root for the runner root", clean) + } + return root, nil +} + +// completeJobScratch removes one terminal job's disposable scratch. It is +// best-effort: a removal failure becomes a bounded job diagnostic, never a +// dispatch failure. It is a no-op on the legacy layout, where no per-job +// scratch exists. +func (d *Dispatcher) completeJobScratch(ctx context.Context, repo, jobID string) { + if d.Storage == nil || !d.RuntimeIdentity.enabled() { + return + } + if err := d.Storage.CompleteJobScratch(ctx, repo, jobID); err != nil { + _ = d.appendDiagnostic(ctx, jobID, "job scratch cleanup: "+safeError(err)) + } +} + func resumeExecutionBinding(command runnercontext.CommandVerb, binding workspace.Binding, session state.PublicSession) (workspace.Binding, error) { if command != runnercontext.CommandResume { return binding, nil @@ -1273,12 +1445,14 @@ func (d *Dispatcher) persistStatusCommentInIntent(ctx context.Context, jobID str func (d *Dispatcher) complete(ctx context.Context, jobID string, command runnercontext.CommandVerb, publicID string, session state.PublicSession, workspaceMeta state.WorkspaceMetadata, dispatch acpx.DispatchResult, terminal state.LifecycleStatus, diagnostics ...string) error { now := d.now() cancelled := false + jobRepo := "" err := d.Store.Update(ctx, func(st *state.RunnerState) error { st.Normalize() current, ok := st.Jobs[jobID] if !ok { return fmt.Errorf("job %q not found", jobID) } + jobRepo = current.Repo if current.Status == state.StatusCancelled { cancelled = true return nil @@ -1343,6 +1517,7 @@ func (d *Dispatcher) complete(ctx context.Context, jobID string, command runnerc if err != nil { return err } + d.completeJobScratch(ctx, jobRepo, jobID) if cancelled { return errDispatchCancelled } @@ -1454,6 +1629,9 @@ func (d *Dispatcher) failWithDispatchMetadata(ctx context.Context, jobID string, if cancelled { return cancelledDuringDispatchResult(jobID), nil } + if failed.ID != "" { + d.completeJobScratch(ctx, failed.Repo, jobID) + } terminalErr := terminalJobFailure(cause) if failed.ID != "" && d.Writeback != nil { _, writebackErr := d.Writeback.Write(ctx, writeback.Request{Job: failed, Status: state.StatusFailed, Phase: phase, @@ -1505,6 +1683,9 @@ func (d *Dispatcher) fail(ctx context.Context, jobID, phase string, cause error) if cancelled { return cancelledDuringDispatchResult(jobID), nil } + if failed.ID != "" { + d.completeJobScratch(ctx, failed.Repo, jobID) + } terminalErr := terminalJobFailure(cause) if failed.ID != "" && d.Writeback != nil { _, writebackErr := d.Writeback.Write(ctx, writeback.Request{Job: failed, Status: state.StatusFailed, Phase: phase, Err: cause}) @@ -1668,6 +1849,10 @@ func (p SandboxRunner) config(req SandboxRequest) (sandbox.Config, string, ghAut cfg.TempXDGConfigHome = firstNonEmpty(req.RuntimeXDGConfigHome, cfg.TempXDGConfigHome) cfg.TempCodexHome = firstNonEmpty(req.RuntimeCodexHome, cfg.TempCodexHome) cfg.AcpxRuntimeDir = firstNonEmpty(req.RuntimeAcpxDir, cfg.AcpxRuntimeDir) + cfg.JobTmpDir = firstNonEmpty(req.JobTmpDir, cfg.JobTmpDir) + cfg.JobGoTmpDir = firstNonEmpty(req.JobGoTmpDir, cfg.JobGoTmpDir) + cfg.JobXDGDataHome = firstNonEmpty(req.JobXDGDataHome, cfg.JobXDGDataHome) + cfg.JobXDGStateHome = firstNonEmpty(req.JobXDGStateHome, cfg.JobXDGStateHome) if strings.TrimSpace(cfg.AcpxRuntimeDir) == "" && strings.TrimSpace(cfg.TempHome) != "" { cfg.AcpxRuntimeDir = filepath.Join(cfg.TempHome, ".acpx", "runtime") } @@ -1721,6 +1906,14 @@ func (p SandboxRunner) config(req SandboxRequest) (sandbox.Config, string, ghAut return sandbox.Config{}, "", ghAuthMirrorResult{}, err } } + for _, dir := range []string{cfg.JobTmpDir, cfg.JobGoTmpDir, cfg.JobXDGDataHome, cfg.JobXDGStateHome} { + if strings.TrimSpace(dir) == "" { + continue + } + if err := os.MkdirAll(dir, 0o700); err != nil { + return sandbox.Config{}, "", ghAuthMirrorResult{}, err + } + } var ghAuthMirror ghAuthMirrorResult if req.ChildProfile != nil { // Darwin's os.UserConfigDir ignores XDG_CONFIG_HOME. In explicit @@ -1735,11 +1928,19 @@ func (p SandboxRunner) config(req SandboxRequest) (sandbox.Config, string, ghAut } // A self-hosted child has no reason to observe hosts.yml or shared gh // state, even if the operator process uses it for legacy GitHub mode. - if err := os.RemoveAll(cfg.TempGHConfigDir); err != nil { + // Content-aware: on the shared runtime home this dir is already empty + // in steady state, and wiping it anyway would race concurrent jobs. + empty, err := dirEmpty(cfg.TempGHConfigDir) + if err != nil { return sandbox.Config{}, "", ghAuthMirror, err } - if err := os.MkdirAll(cfg.TempGHConfigDir, 0o700); err != nil { - return sandbox.Config{}, "", ghAuthMirror, err + if !empty { + if err := os.RemoveAll(cfg.TempGHConfigDir); err != nil { + return sandbox.Config{}, "", ghAuthMirror, err + } + if err := os.MkdirAll(cfg.TempGHConfigDir, 0o700); err != nil { + return sandbox.Config{}, "", ghAuthMirror, err + } } } else { var err error @@ -1844,28 +2045,70 @@ func materializeChildProfile(xdgConfigHome string, profile clientauth.Profile) e if err != nil { return err } - temporary, err := os.CreateTemp(dir, ".profiles-*") + return writeFileAtomic(filepath.Join(dir, "profiles.json"), append(data, '\n'), 0o600) +} + +// dirEmpty reports whether dir exists and contains no entries. A missing dir +// counts as empty: there is nothing to wipe. +func dirEmpty(dir string) (bool, error) { + entries, err := os.ReadDir(dir) + if errors.Is(err, os.ErrNotExist) { + return true, nil + } if err != nil { + return false, err + } + return len(entries) == 0, nil +} + +// writeFileAtomic installs data at path through a temp file in the destination +// directory plus rename, so a concurrent reader of the shared runtime home +// never observes a partial mirror. An existing identical regular file is left +// untouched (mode enforced): steady-state refreshes must not churn the shared +// home. +func writeFileAtomic(path string, data []byte, mode os.FileMode) error { + if info, err := os.Lstat(path); err == nil && info.Mode().IsRegular() { + if existing, readErr := os.ReadFile(path); readErr == nil && bytes.Equal(existing, data) { + if info.Mode().Perm() != mode { + return os.Chmod(path, mode) + } + return nil + } + } + if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil { return err } - temporaryName := temporary.Name() - defer os.Remove(temporaryName) - if err := temporary.Chmod(0o600); err != nil { - temporary.Close() + tmp, err := os.CreateTemp(filepath.Dir(path), ".issue-spec-mirror-*") + if err != nil { return err } - if _, err := temporary.Write(append(data, '\n')); err != nil { - temporary.Close() + tmpName := tmp.Name() + cleanup := true + defer func() { + if cleanup { + _ = os.Remove(tmpName) + } + }() + if err := tmp.Chmod(mode); err != nil { + _ = tmp.Close() + return err + } + if _, err := tmp.Write(data); err != nil { + _ = tmp.Close() return err } - if err := temporary.Sync(); err != nil { - temporary.Close() + if err := tmp.Sync(); err != nil { + _ = tmp.Close() return err } - if err := temporary.Close(); err != nil { + if err := tmp.Close(); err != nil { return err } - return os.Rename(temporaryName, filepath.Join(dir, "profiles.json")) + if err := os.Rename(tmpName, path); err != nil { + return err + } + cleanup = false + return nil } func requestReadOnlyBinds(req SandboxRequest, acpxBinary string, lookPath func(string) (string, error)) ([]string, []string, string, error) { @@ -2336,35 +2579,14 @@ func copyGHConfigDir(source, dest string) error { if entry.IsDir() { return os.MkdirAll(target, 0o700) } - info, err := os.Lstat(target) - if err != nil && !errors.Is(err, os.ErrNotExist) { - return err - } - if err == nil { - if info.IsDir() { - return fmt.Errorf("target %s is a directory", target) - } - if info.Mode()&os.ModeSymlink != 0 { - if err := os.Remove(target); err != nil { - return err - } - } else if info.Mode().IsRegular() { - if err := os.Chmod(target, 0o600); err != nil { - return err - } - } + if info, err := os.Lstat(target); err == nil && info.IsDir() { + return fmt.Errorf("target %s is a directory", target) } data, err := os.ReadFile(path) if err != nil { return err } - if err := os.MkdirAll(filepath.Dir(target), 0o700); err != nil { - return err - } - if err := os.WriteFile(target, data, 0o600); err != nil { - return err - } - return os.Chmod(target, 0o600) + return writeFileAtomic(target, data, 0o600) }) } @@ -2456,24 +2678,14 @@ func copyLimitedCodexConfig(source, dest string) error { return err } data = sanitizeCodexRuntimeFile(name, data) - if targetInfo, err := os.Lstat(targetPath); err == nil { - if targetInfo.IsDir() { - return fmt.Errorf("target %s is a directory", targetPath) - } - if err := os.Remove(targetPath); err != nil { - return err - } - } else if !errors.Is(err, os.ErrNotExist) { - return err + if targetInfo, err := os.Lstat(targetPath); err == nil && targetInfo.IsDir() { + return fmt.Errorf("target %s is a directory", targetPath) } mode := info.Mode().Perm() if mode == 0 { mode = 0o600 } - if err := os.WriteFile(targetPath, data, mode); err != nil { - return err - } - if err := os.Chmod(targetPath, mode); err != nil { + if err := writeFileAtomic(targetPath, data, mode); err != nil { return err } } @@ -2572,17 +2784,11 @@ func copyLimitedFiles(source, dest string, names []string) error { if err != nil { return err } - if err := os.MkdirAll(filepath.Dir(targetPath), 0o700); err != nil { - return err - } mode := info.Mode().Perm() if mode == 0 { mode = 0o600 } - if err := os.WriteFile(targetPath, data, mode); err != nil { - return err - } - if err := os.Chmod(targetPath, mode); err != nil { + if err := writeFileAtomic(targetPath, data, mode); err != nil { return err } } @@ -3168,6 +3374,18 @@ func tempPaths(meta sandbox.EnvMetadata) map[string]string { if meta.CodexHome != "" { out["CODEX_HOME"] = meta.CodexHome } + if meta.TmpDir != "" { + out["TMPDIR"] = meta.TmpDir + } + if meta.GoTmpDir != "" { + out["GOTMPDIR"] = meta.GoTmpDir + } + if meta.XDGDataHome != "" { + out["XDG_DATA_HOME"] = meta.XDGDataHome + } + if meta.XDGStateHome != "" { + out["XDG_STATE_HOME"] = meta.XDGStateHome + } if len(out) == 0 { return nil } diff --git a/internal/commentrunner/jobs/reconcile.go b/internal/commentrunner/jobs/reconcile.go index 6e1a73e1..382f4d39 100644 --- a/internal/commentrunner/jobs/reconcile.go +++ b/internal/commentrunner/jobs/reconcile.go @@ -386,14 +386,39 @@ func (d *Dispatcher) coordinatorForStoredJob(ctx context.Context, job state.Job) if err != nil { return nil, err } - runtimePaths, err := stableSessionRuntimePaths(workspacePath, job.Repo, publicID) - if err != nil { - return nil, err + var runtimePaths sessionRuntimePaths + var sharedHome storage.RuntimeHomePaths + if d.RuntimeIdentity.enabled() { + // Restart reconcile/cancel reuses the runner-scoped shared HOME so the + // coordinator sees the exact runtime the job was dispatched with. It + // never prepares per-job scratch: no new dispatch runs here. + workspaceRoot, rootErr := runnerWorkspaceRootFor(workspacePath) + if rootErr != nil { + return nil, rootErr + } + home, homeErr := storage.PrepareRuntimeHome(workspaceRoot, d.RuntimeIdentity.scope(job.Repo)) + if homeErr != nil { + return nil, homeErr + } + sharedHome = home + runtimePaths = sessionRuntimePaths{ + home: home.Home, + ghConfigDir: home.GHConfigDir, + xdgConfigHome: home.XDGConfigHome, + codexHome: home.CodexHome, + acpxRuntimeDir: home.AcpxRuntimeDir, + } + } else { + var pathErr error + runtimePaths, pathErr = stableSessionRuntimePaths(workspacePath, job.Repo, publicID) + if pathErr != nil { + return nil, pathErr + } } // Best-effort touch: a recording failure must not kill already running // work, but it is diagnosed so startup/explicit reconcile can repair. if d.Storage != nil { - if err := d.Storage.RecordSessionResources(ctx, job.Repo, publicID, workspacePath); err != nil { + if err := d.recordStoredJobRuntime(ctx, job, publicID, workspacePath, sharedHome); err != nil { _ = d.appendDiagnostic(ctx, job.ID, "storage recording: "+safeError(err)) } } @@ -420,6 +445,19 @@ func (d *Dispatcher) coordinatorForStoredJob(ctx context.Context, job state.Job) return coordinator, err } +// recordStoredJobRuntime re-records a stored job's physical identities during +// restart reconcile: the shared home plus session PROCESS pool on the shared +// layout, the per-session runtime and pool on the legacy layout. +func (d *Dispatcher) recordStoredJobRuntime(ctx context.Context, job state.Job, publicID, workspacePath string, sharedHome storage.RuntimeHomePaths) error { + if !d.RuntimeIdentity.enabled() { + return d.Storage.RecordSessionResources(ctx, job.Repo, publicID, workspacePath) + } + if err := d.Storage.RecordRuntimeHome(ctx, d.RuntimeIdentity.scope(job.Repo), sharedHome); err != nil { + return err + } + return d.Storage.RecordSessionProcessPool(ctx, job.Repo, publicID, workspacePath) +} + func (d *Dispatcher) applyReconcile(ctx context.Context, job state.Job, previous state.LifecycleStatus, reconciled acpx.TurnReconcileResult) (ReconcileJob, error) { diagnostic := strings.TrimSpace(reconciled.Diagnostics) status := state.LifecycleStatus(reconciled.Status) @@ -514,6 +552,7 @@ func (d *Dispatcher) recoveredTerminal(ctx context.Context, job state.Job, previ }); err != nil { return ReconcileJob{}, err } + d.completeJobScratch(ctx, final.Repo, final.ID) d.releaseLock(ctx, final.ID, lock) req := writeback.Request{ Job: final, diff --git a/internal/commentrunner/jobs/storage_integration_test.go b/internal/commentrunner/jobs/storage_integration_test.go index eacbb1a3..ed5fa608 100644 --- a/internal/commentrunner/jobs/storage_integration_test.go +++ b/internal/commentrunner/jobs/storage_integration_test.go @@ -20,10 +20,34 @@ type recordedSessionResource struct { workspace string } +type recordedRuntimeHome struct { + scope storage.RuntimeScope + paths storage.RuntimeHomePaths +} + +type recordedJobScratch struct { + repo string + jobID string + path string +} + +type recordedScratchCompletion struct { + repo string + jobID string +} + type fakeStorage struct { admitErr error recordErr error recordCalls []recordedSessionResource + poolErr error + poolCalls []recordedSessionResource + homeErr error + homeCalls []recordedRuntimeHome + scratchErr error + scratchCalls []recordedJobScratch + completeErr error + completeCalls []recordedScratchCompletion admitCalls int reconcileCalls int reconcileApply []bool @@ -41,6 +65,26 @@ func (f *fakeStorage) RecordSessionResources(_ context.Context, repo, publicSess return f.recordErr } +func (f *fakeStorage) RecordSessionProcessPool(_ context.Context, repo, publicSessionID, workspacePath string) error { + f.poolCalls = append(f.poolCalls, recordedSessionResource{repo: repo, sid: publicSessionID, workspace: workspacePath}) + return f.poolErr +} + +func (f *fakeStorage) RecordRuntimeHome(_ context.Context, scope storage.RuntimeScope, paths storage.RuntimeHomePaths) error { + f.homeCalls = append(f.homeCalls, recordedRuntimeHome{scope: scope, paths: paths}) + return f.homeErr +} + +func (f *fakeStorage) RecordJobScratch(_ context.Context, repo, jobID, path string) error { + f.scratchCalls = append(f.scratchCalls, recordedJobScratch{repo: repo, jobID: jobID, path: path}) + return f.scratchErr +} + +func (f *fakeStorage) CompleteJobScratch(_ context.Context, repo, jobID string) error { + f.completeCalls = append(f.completeCalls, recordedScratchCompletion{repo: repo, jobID: jobID}) + return f.completeErr +} + func (f *fakeStorage) ReconcileStorage(_ context.Context, apply, _ bool) (storage.Report, error) { f.reconcileCalls++ f.reconcileApply = append(f.reconcileApply, apply) From 1cace105ee8c4ed5154b9e3700f984edf7aed9c0 Mon Sep 17 00:00:00 2001 From: johnlanni Date: Sat, 8 Aug 2026 22:41:10 +0800 Subject: [PATCH 08/17] test(runner): cover shared-layout dispatch, fail-closed recording, and mirror atomicity Add dispatcher tests for the runner-scoped shared layout: two sessions of one repo receive the identical runtime HOME with distinct per-job scratch, a different repo receives a different home below the same .runner-home root, and the zero runtime identity keeps the legacy .sessions/ layout untouched. Fail-closed recording is pinned in order (home, pool, scratch) with sandbox Prepare never reached on failure, and every terminal transition (success, dispatch failure, queued cancellation, confirmed running cancellation) completes the job scratch exactly once. Mirror tests hammer the shared home with concurrent gh/codex refreshes while a reader validates no torn file is ever observed, and pin the skip-unchanged behavior so steady-state refreshes perform no writes. Signed-off-by: johnlanni --- .../jobs/shared_layout_dispatch_test.go | 704 ++++++++++++++++++ 1 file changed, 704 insertions(+) create mode 100644 internal/commentrunner/jobs/shared_layout_dispatch_test.go diff --git a/internal/commentrunner/jobs/shared_layout_dispatch_test.go b/internal/commentrunner/jobs/shared_layout_dispatch_test.go new file mode 100644 index 00000000..5f3529d5 --- /dev/null +++ b/internal/commentrunner/jobs/shared_layout_dispatch_test.go @@ -0,0 +1,704 @@ +package jobs + +import ( + "context" + "errors" + "fmt" + "os" + "path/filepath" + "strings" + "sync" + "testing" + "time" + + "github.com/higress-group/issue-spec/internal/acpx" + "github.com/higress-group/issue-spec/internal/commentrunner/state" + "github.com/higress-group/issue-spec/internal/commentrunner/storage" + "github.com/higress-group/issue-spec/internal/workspace" +) + +// sharedLayoutIdentity is the dispatcher runtime identity used by the shared +// layout tests: builtin GitHub profile shape (empty realm). +func sharedLayoutIdentity() RuntimeIdentity { + return RuntimeIdentity{Hostname: "github.com", Runner: "runner-shared"} +} + +// sharedLayoutBinding builds a session-clone binding rooted below root so the +// runner workspace root (the parent of the clone) is root itself. +func sharedLayoutBinding(t *testing.T, root, id, repo string) workspace.Binding { + t.Helper() + path := filepath.Join(root, id) + if err := os.MkdirAll(path, 0o700); err != nil { + t.Fatal(err) + } + return workspace.Binding{ + Workspace: state.WorkspaceMetadata{ID: id, Path: path, Repo: repo, CloneURL: "https://github.com/" + repo + ".git", Branch: "issue-spec-" + id, Ref: "main", RepositoryBinding: sharedLayoutRepoBinding(repo)}, + AcpxWorkingDirectory: path, + SandboxWorkspacePath: path, + } +} + +func sharedLayoutRepoBinding(repo string) state.RepositoryBindingSnapshot { + binding := testRepositoryBinding() + binding.IssueRepositoryKey = repo + binding.ExternalRepositoryID = repo + binding.CloneURL = "https://github.com/" + repo + ".git" + binding.WebURL = "https://github.com/" + repo + return binding +} + +// perRepoResolver resolves the requested repo verbatim so one dispatcher can +// dispatch jobs of different repos into different runtime scopes. +type perRepoResolver struct{} + +func (perRepoResolver) ResolveRepository(_ context.Context, repo string) (RepositoryInfo, error) { + binding := sharedLayoutRepoBinding(repo) + return RepositoryInfo{Repo: repo, CloneURL: binding.CloneURL, DefaultBranch: "main", Ref: "main", Binding: binding}, nil +} + +func seedSharedLayoutJob(t *testing.T, store *memoryStore, id, repo string, now time.Time) { + t.Helper() + seedQueuedJob(t, store, state.Job{ + ID: id, + Repo: repo, + IssueNumber: 30, + SessionCreatorLogin: "alice", + TriggeringUserLogin: "alice", + TriggerCommentID: 42, + CommandID: "cmd-" + id, + CommandName: "new", + CommandPrompt: "implement", + CommandIdempotencyKey: "key-" + id, + StatusWritebackKey: "status-" + id, + CreatedAt: now, + FirstObservedComment: state.SeenComment{ + Repo: repo, IssueNumber: 30, CommentID: 42, AuthorLogin: "alice", + HTMLURL: "https://github.com/" + repo + "/issues/30#issuecomment-42", + }, + }) +} + +func newSharedLayoutDispatcher(store *memoryStore, workspaces *fakeWorkspaces, coordinator *fakeCoordinator, fake *fakeStorage, now time.Time) (*Dispatcher, *fakeSandbox) { + dispatcher := testDispatcher(store, workspaces, coordinator, &fakeWriteback{}, now) + dispatcher.Repositories = perRepoResolver{} + dispatcher.Storage = fake + dispatcher.RuntimeIdentity = sharedLayoutIdentity() + return dispatcher, dispatcher.Sandbox.(*fakeSandbox) +} + +// TestSharedLayoutDispatchSharesRuntimeHomeAcrossSessions is the core shared +// layout contract: two sessions of one repo on one runner identity dispatch +// onto the identical runner-scoped runtime HOME while each job receives its +// own disposable scratch, and no legacy .sessions runtime is created. +func TestSharedLayoutDispatchSharesRuntimeHomeAcrossSessions(t *testing.T) { + store := newMemoryStore() + now := time.Date(2026, 7, 6, 9, 0, 0, 0, time.UTC) + root, err := filepath.EvalSymlinks(t.TempDir()) + if err != nil { + t.Fatal(err) + } + workspaces := &fakeWorkspaces{bindings: map[string]workspace.Binding{ + "job-aaaaaaaaaaaaaaaa": sharedLayoutBinding(t, root, "ws-shared-a", "o/r"), + "job-bbbbbbbbbbbbbbbb": sharedLayoutBinding(t, root, "ws-shared-b", "o/r"), + }} + coordinator := &fakeCoordinator{newResult: dispatchResult("ps-first", "rec-first", "turn-first", completedSummary())} + fake := &fakeStorage{} + dispatcher, sandbox := newSharedLayoutDispatcher(store, workspaces, coordinator, fake, now) + + seedSharedLayoutJob(t, store, "job-aaaaaaaaaaaaaaaa", "o/r", now) + result, err := dispatcher.RunNext(context.Background()) + if err != nil || !result.Executed || result.Status != state.StatusCompleted { + t.Fatalf("first dispatch result=%+v err=%v", result, err) + } + + dispatcher.PublicSessionID = func() (string, error) { return "ps-second", nil } + coordinator.newResult = dispatchResult("ps-second", "rec-second", "turn-second", completedSummary()) + seedSharedLayoutJob(t, store, "job-bbbbbbbbbbbbbbbb", "o/r", now.Add(time.Minute)) + result, err = dispatcher.RunNext(context.Background()) + if err != nil || !result.Executed || result.Status != state.StatusCompleted { + t.Fatalf("second dispatch result=%+v err=%v", result, err) + } + + if len(sandbox.requests) != 2 { + t.Fatalf("sandbox requests = %d, want 2", len(sandbox.requests)) + } + first, second := sandbox.requests[0], sandbox.requests[1] + + scope := sharedLayoutIdentity().scope("o/r") + homeRoot, err := storage.RuntimeHomeRoot(root, scope) + if err != nil { + t.Fatal(err) + } + wantHome := filepath.Join(homeRoot, "home") + if first.RuntimeHome != wantHome || second.RuntimeHome != wantHome { + t.Fatalf("shared runtime home mismatch: first=%q second=%q want=%q", first.RuntimeHome, second.RuntimeHome, wantHome) + } + for _, req := range []SandboxRequest{first, second} { + if req.RuntimeGHConfigDir != filepath.Join(homeRoot, "gh") || + req.RuntimeXDGConfigHome != filepath.Join(homeRoot, "xdg") || + req.RuntimeCodexHome != filepath.Join(homeRoot, "codex") || + req.RuntimeAcpxDir != filepath.Join(homeRoot, "acpx-runtime") { + t.Fatalf("runtime home subdirs do not anchor at the shared root %q: %+v", homeRoot, req) + } + } + + scratchRootA, err := storage.JobScratchRoot(root, "job-aaaaaaaaaaaaaaaa") + if err != nil { + t.Fatal(err) + } + scratchRootB, err := storage.JobScratchRoot(root, "job-bbbbbbbbbbbbbbbb") + if err != nil { + t.Fatal(err) + } + if first.JobTmpDir != filepath.Join(scratchRootA, "tmp") || second.JobTmpDir != filepath.Join(scratchRootB, "tmp") { + t.Fatalf("job tmp dirs are not the per-job scratch: first=%q second=%q", first.JobTmpDir, second.JobTmpDir) + } + if first.JobTmpDir == second.JobTmpDir || first.JobGoTmpDir == second.JobGoTmpDir || + first.JobXDGDataHome == second.JobXDGDataHome || first.JobXDGStateHome == second.JobXDGStateHome { + t.Fatalf("jobs of one repo must receive distinct scratch: first=%+v second=%+v", first, second) + } + for _, dir := range []string{first.JobTmpDir, first.JobGoTmpDir, first.JobXDGDataHome, first.JobXDGStateHome} { + if info, statErr := os.Stat(dir); statErr != nil || !info.IsDir() { + t.Fatalf("scratch dir %q not prepared: info=%v err=%v", dir, info, statErr) + } + } + + // Recording order and shape: shared home + process pool + job scratch per + // dispatch; the legacy session runtime record is never used. + if len(fake.recordCalls) != 0 { + t.Fatalf("legacy RecordSessionResources must not run on the shared layout: %+v", fake.recordCalls) + } + if len(fake.homeCalls) != 2 || len(fake.poolCalls) != 2 || len(fake.scratchCalls) != 2 { + t.Fatalf("shared layout recording calls: home=%d pool=%d scratch=%d, want 2 each", + len(fake.homeCalls), len(fake.poolCalls), len(fake.scratchCalls)) + } + if fake.homeCalls[0].scope != scope || fake.homeCalls[1].scope != scope { + t.Fatalf("runtime home scopes = %+v, want %+v", fake.homeCalls, scope) + } + if fake.scratchCalls[0].jobID != "job-aaaaaaaaaaaaaaaa" || fake.scratchCalls[1].jobID != "job-bbbbbbbbbbbbbbbb" || + fake.scratchCalls[0].path != scratchRootA || fake.scratchCalls[1].path != scratchRootB { + t.Fatalf("scratch recordings = %+v", fake.scratchCalls) + } + + // Both jobs completed: both scratch trees are completed and removed. + if len(fake.completeCalls) != 2 { + t.Fatalf("complete calls = %+v, want 2", fake.completeCalls) + } + + // The shared layout never materializes the legacy per-session runtime root. + if _, err := os.Lstat(filepath.Join(root, ".sessions")); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("legacy .sessions root must not exist on the shared layout: %v", err) + } +} + +// TestSharedLayoutRuntimeHomeDiffersAcrossRepos proves the runtime HOME is +// scoped per repo: a job of another repo on the same runner identity receives +// a different shared home below the same .runner-home root. +func TestSharedLayoutRuntimeHomeDiffersAcrossRepos(t *testing.T) { + store := newMemoryStore() + now := time.Date(2026, 7, 6, 9, 30, 0, 0, time.UTC) + root, err := filepath.EvalSymlinks(t.TempDir()) + if err != nil { + t.Fatal(err) + } + workspaces := &fakeWorkspaces{bindings: map[string]workspace.Binding{ + "job-aaaaaaaaaaaaaaaa": sharedLayoutBinding(t, root, "ws-repo-a", "o/r"), + "job-cccccccccccccccc": sharedLayoutBinding(t, root, "ws-repo-c", "o/r2"), + }} + coordinator := &fakeCoordinator{newResult: dispatchResult("ps-repo", "rec-repo", "turn-repo", completedSummary())} + dispatcher, sandbox := newSharedLayoutDispatcher(store, workspaces, coordinator, &fakeStorage{}, now) + + seedSharedLayoutJob(t, store, "job-aaaaaaaaaaaaaaaa", "o/r", now) + if result, err := dispatcher.RunNext(context.Background()); err != nil || !result.Executed { + t.Fatalf("first repo dispatch result=%+v err=%v", result, err) + } + dispatcher.PublicSessionID = func() (string, error) { return "ps-repo-two", nil } + seedSharedLayoutJob(t, store, "job-cccccccccccccccc", "o/r2", now.Add(time.Minute)) + if result, err := dispatcher.RunNext(context.Background()); err != nil || !result.Executed { + t.Fatalf("second repo dispatch result=%+v err=%v", result, err) + } + + if len(sandbox.requests) != 2 { + t.Fatalf("sandbox requests = %d, want 2", len(sandbox.requests)) + } + first, second := sandbox.requests[0], sandbox.requests[1] + if first.RuntimeHome == "" || first.RuntimeHome == second.RuntimeHome { + t.Fatalf("different repos must not share a runtime home: first=%q second=%q", first.RuntimeHome, second.RuntimeHome) + } + wantSecond, err := storage.RuntimeHomeRoot(root, sharedLayoutIdentity().scope("o/r2")) + if err != nil { + t.Fatal(err) + } + if second.RuntimeHome != filepath.Join(wantSecond, "home") { + t.Fatalf("second repo home = %q, want %q", second.RuntimeHome, filepath.Join(wantSecond, "home")) + } + if filepath.Dir(filepath.Dir(first.RuntimeHome)) != filepath.Join(root, storage.RunnerHomesDirName) || + filepath.Dir(filepath.Dir(second.RuntimeHome)) != filepath.Join(root, storage.RunnerHomesDirName) { + t.Fatalf("both homes must anchor below %q: first=%q second=%q", storage.RunnerHomesDirName, first.RuntimeHome, second.RuntimeHome) + } +} + +// TestZeroRuntimeIdentityKeepsLegacySessionLayout pins the zero-identity +// behavior: existing wiring without a runner runtime identity keeps the +// per-session .sessions/ runtime and receives no job scratch. +func TestZeroRuntimeIdentityKeepsLegacySessionLayout(t *testing.T) { + store := newMemoryStore() + now := time.Date(2026, 7, 6, 10, 0, 0, 0, time.UTC) + root, err := filepath.EvalSymlinks(t.TempDir()) + if err != nil { + t.Fatal(err) + } + workspaces := &fakeWorkspaces{binding: sharedLayoutBinding(t, root, "ws-legacy", "o/r")} + coordinator := &fakeCoordinator{newResult: dispatchResult("ps-legacy", "rec-legacy", "turn-legacy", completedSummary())} + fake := &fakeStorage{} + dispatcher := testDispatcher(store, workspaces, coordinator, &fakeWriteback{}, now) + dispatcher.Storage = fake + sandbox := dispatcher.Sandbox.(*fakeSandbox) + + seedSharedLayoutJob(t, store, "job-legacy-shared", "o/r", now) + result, err := dispatcher.RunNext(context.Background()) + if err != nil || !result.Executed || result.Status != state.StatusCompleted { + t.Fatalf("legacy dispatch result=%+v err=%v", result, err) + } + if len(sandbox.requests) != 1 { + t.Fatalf("sandbox requests = %d, want 1", len(sandbox.requests)) + } + req := sandbox.requests[0] + legacyRoot, err := storage.SessionRuntimeRoot(filepath.Join(root, "ws-legacy"), "o/r", "ps-generated") + if err != nil { + t.Fatal(err) + } + if req.RuntimeHome != filepath.Join(legacyRoot, "home") { + t.Fatalf("legacy runtime home = %q, want %q", req.RuntimeHome, filepath.Join(legacyRoot, "home")) + } + if !strings.Contains(req.RuntimeHome, string(filepath.Separator)+storage.SessionsDirName+string(filepath.Separator)) { + t.Fatalf("legacy runtime home must live below .sessions: %q", req.RuntimeHome) + } + if req.JobTmpDir != "" || req.JobGoTmpDir != "" || req.JobXDGDataHome != "" || req.JobXDGStateHome != "" { + t.Fatalf("legacy layout must not receive job scratch: %+v", req) + } + if len(fake.recordCalls) != 1 || len(fake.homeCalls) != 0 || len(fake.scratchCalls) != 0 || len(fake.poolCalls) != 0 { + t.Fatalf("legacy recording must use RecordSessionResources only: record=%d home=%d pool=%d scratch=%d", + len(fake.recordCalls), len(fake.homeCalls), len(fake.poolCalls), len(fake.scratchCalls)) + } + if len(fake.completeCalls) != 0 { + t.Fatalf("legacy layout has no scratch to complete: %+v", fake.completeCalls) + } + if _, err := os.Lstat(filepath.Join(root, storage.RunnerHomesDirName)); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("shared runner-home root must not exist on the legacy layout: %v", err) + } +} + +// TestSharedLayoutRecordingFailureBlocksDispatchBeforeSandbox proves the +// fail-closed recording order: a runtime home, process pool, or job scratch +// recording failure fails the dispatch before the sandbox ever sees the +// runtime. +func TestSharedLayoutRecordingFailureBlocksDispatchBeforeSandbox(t *testing.T) { + tests := []struct { + name string + configure func(*fakeStorage) + assert func(t *testing.T, fake *fakeStorage) + }{ + { + name: "runtime home", + configure: func(f *fakeStorage) { f.homeErr = errors.New("sidecar home write failed") }, + assert: func(t *testing.T, fake *fakeStorage) { + t.Helper() + if len(fake.homeCalls) != 1 || len(fake.poolCalls) != 0 || len(fake.scratchCalls) != 0 { + t.Fatalf("home failure must stop before pool/scratch recording: home=%d pool=%d scratch=%d", + len(fake.homeCalls), len(fake.poolCalls), len(fake.scratchCalls)) + } + }, + }, + { + name: "process pool", + configure: func(f *fakeStorage) { f.poolErr = errors.New("sidecar pool write failed") }, + assert: func(t *testing.T, fake *fakeStorage) { + t.Helper() + if len(fake.homeCalls) != 1 || len(fake.poolCalls) != 1 || len(fake.scratchCalls) != 0 { + t.Fatalf("pool failure must stop before scratch recording: home=%d pool=%d scratch=%d", + len(fake.homeCalls), len(fake.poolCalls), len(fake.scratchCalls)) + } + }, + }, + { + name: "job scratch", + configure: func(f *fakeStorage) { f.scratchErr = errors.New("sidecar scratch write failed") }, + assert: func(t *testing.T, fake *fakeStorage) { + t.Helper() + if len(fake.homeCalls) != 1 || len(fake.poolCalls) != 1 || len(fake.scratchCalls) != 1 { + t.Fatalf("scratch failure follows home+pool recording: home=%d pool=%d scratch=%d", + len(fake.homeCalls), len(fake.poolCalls), len(fake.scratchCalls)) + } + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + store := newMemoryStore() + now := time.Date(2026, 7, 6, 10, 30, 0, 0, time.UTC) + root, err := filepath.EvalSymlinks(t.TempDir()) + if err != nil { + t.Fatal(err) + } + workspaces := &fakeWorkspaces{binding: sharedLayoutBinding(t, root, "ws-failclosed", "o/r")} + coordinator := &fakeCoordinator{newResult: dispatchResult("ps-failclosed", "rec-failclosed", "turn-failclosed", completedSummary())} + fake := &fakeStorage{} + tt.configure(fake) + dispatcher, sandbox := newSharedLayoutDispatcher(store, workspaces, coordinator, fake, now) + + seedSharedLayoutJob(t, store, "job-dddddddddddddddd", "o/r", now) + result, err := dispatcher.RunNext(context.Background()) + if err == nil { + t.Fatalf("recording failure must fail the dispatch: %+v", result) + } + if got := loadState(t, store).Jobs["job-dddddddddddddddd"].Status; got != state.StatusFailed { + t.Fatalf("recording failure must fail the job fail-closed, got %q", got) + } + if len(sandbox.requests) != 0 { + t.Fatalf("sandbox Prepare must not run when recording fails: %+v", sandbox.requests) + } + tt.assert(t, fake) + }) + } +} + +// TestSharedLayoutTerminalTransitionsCompleteJobScratch covers every terminal +// path that must retire the job's disposable scratch: successful completion, +// dispatch failure, cancellation before dispatch, and confirmed cancellation +// of a running job. +func TestSharedLayoutTerminalTransitionsCompleteJobScratch(t *testing.T) { + now := time.Date(2026, 7, 6, 11, 0, 0, 0, time.UTC) + + t.Run("success", func(t *testing.T) { + store := newMemoryStore() + root, err := filepath.EvalSymlinks(t.TempDir()) + if err != nil { + t.Fatal(err) + } + workspaces := &fakeWorkspaces{binding: sharedLayoutBinding(t, root, "ws-terminal-ok", "o/r")} + coordinator := &fakeCoordinator{newResult: dispatchResult("ps-terminal-ok", "rec-terminal-ok", "turn-terminal-ok", completedSummary())} + fake := &fakeStorage{} + dispatcher, _ := newSharedLayoutDispatcher(store, workspaces, coordinator, fake, now) + + seedSharedLayoutJob(t, store, "job-eeeeeeeeeeeeeeee", "o/r", now) + result, err := dispatcher.RunNext(context.Background()) + if err != nil || !result.Executed || result.Status != state.StatusCompleted { + t.Fatalf("dispatch result=%+v err=%v", result, err) + } + if len(fake.completeCalls) != 1 || fake.completeCalls[0].jobID != "job-eeeeeeeeeeeeeeee" || fake.completeCalls[0].repo != "o/r" { + t.Fatalf("successful terminal completion must complete the job scratch exactly once: %+v", fake.completeCalls) + } + }) + + t.Run("dispatch failure", func(t *testing.T) { + store := newMemoryStore() + root, err := filepath.EvalSymlinks(t.TempDir()) + if err != nil { + t.Fatal(err) + } + workspaces := &fakeWorkspaces{binding: sharedLayoutBinding(t, root, "ws-terminal-fail", "o/r")} + coordinator := &fakeCoordinator{newErr: errors.New("coordinator exploded")} + fake := &fakeStorage{} + dispatcher, _ := newSharedLayoutDispatcher(store, workspaces, coordinator, fake, now) + + seedSharedLayoutJob(t, store, "job-ffffffffffffffff", "o/r", now) + if _, err := dispatcher.RunNext(context.Background()); err == nil { + t.Fatalf("terminal coordinator failure must surface an error") + } + if got := loadState(t, store).Jobs["job-ffffffffffffffff"].Status; got != state.StatusFailed { + t.Fatalf("job status = %q, want failed", got) + } + if len(fake.completeCalls) != 1 || fake.completeCalls[0].jobID != "job-ffffffffffffffff" { + t.Fatalf("failed terminal completion must complete the job scratch exactly once: %+v", fake.completeCalls) + } + }) + + t.Run("cancel queued", func(t *testing.T) { + store := newMemoryStore() + root, err := filepath.EvalSymlinks(t.TempDir()) + if err != nil { + t.Fatal(err) + } + workspaceMeta := sharedLayoutBinding(t, root, "ws-cancel-queued", "o/r").Workspace + seedState(t, store, func(st *state.RunnerState) error { + if err := st.UpsertWorkspace(workspaceMeta); err != nil { + return err + } + if err := st.UpsertPublicSession(state.PublicSession{ + Repo: "o/r", PublicSessionID: "ps-cancel-queued", IssueNumber: 30, CreatorLogin: "alice", + AcpxRecordID: "rec-cancel-queued", + Status: state.StatusCompleted, Workspace: workspaceMeta, RepositoryBinding: sharedLayoutRepoBinding("o/r"), + Queue: state.SessionQueue{AcceptedSequence: 1, PendingJobIDs: []string{"job-1111111111111111"}}, + CreatedAt: now, LastUsedAt: now, + }); err != nil { + return err + } + if err := st.UpsertJob(state.Job{ + ID: "job-1111111111111111", Repo: "o/r", IssueNumber: 30, PublicSessionID: "ps-cancel-queued", + CommandName: "resume", CommandPrompt: "queued behind", CommandIdempotencyKey: "key-cancel-queued", + Status: state.StatusQueued, CreatedAt: now, Workspace: workspaceMeta, + RepositoryBinding: sharedLayoutRepoBinding("o/r"), + }); err != nil { + return err + } + return st.UpsertCancellation(state.Cancellation{ + ID: "cancel-queued-scratch", IdempotencyKey: "cancel-key-queued-scratch", Repo: "o/r", IssueNumber: 30, + TriggerCommentID: 909, CancelingUserLogin: "bob", TargetPublicSessionID: "ps-cancel-queued", + Status: state.StatusQueued, CreatedAt: now, + }) + }) + fake := &fakeStorage{} + dispatcher, _ := newSharedLayoutDispatcher(store, &fakeWorkspaces{}, &fakeCoordinator{}, fake, now) + + result, err := dispatcher.DrainCancellations(context.Background(), 0) + if err != nil { + t.Fatalf("DrainCancellations: %v", err) + } + if !result.Executed || result.JobID != "job-1111111111111111" || result.Status != state.StatusCancelled { + t.Fatalf("cancel result = %+v", result) + } + if len(fake.completeCalls) != 1 || fake.completeCalls[0].jobID != "job-1111111111111111" { + t.Fatalf("queued cancellation must complete the job scratch: %+v", fake.completeCalls) + } + }) + + t.Run("cancel running", func(t *testing.T) { + store := newMemoryStore() + root, err := filepath.EvalSymlinks(t.TempDir()) + if err != nil { + t.Fatal(err) + } + workspaceMeta := sharedLayoutBinding(t, root, "ws-cancel-running", "o/r").Workspace + seedState(t, store, func(st *state.RunnerState) error { + if err := st.UpsertWorkspace(workspaceMeta); err != nil { + return err + } + if err := st.UpsertJob(state.Job{ + ID: "job-2222222222222222", Repo: "o/r", IssueNumber: 30, PublicSessionID: "ps-cancel-running", + CoordinatorKind: "codex", SessionCreatorLogin: "alice", TriggeringUserLogin: "alice", + TriggerCommentID: 910, StatusWritebackKey: "status-cancel-running", + Status: state.StatusRunning, CreatedAt: now, UpdatedAt: now, Workspace: workspaceMeta, + RepositoryBinding: sharedLayoutRepoBinding("o/r"), + DispatchIntent: state.DispatchIntent{RunnerJobID: "job-2222222222222222", PublicSessionID: "ps-cancel-running"}, + }); err != nil { + return err + } + if err := st.UpsertPublicSession(state.PublicSession{ + Repo: "o/r", PublicSessionID: "ps-cancel-running", IssueNumber: 30, CreatorLogin: "alice", + Status: state.StatusRunning, Workspace: workspaceMeta, RepositoryBinding: sharedLayoutRepoBinding("o/r"), + CreatedAt: now, LastUsedAt: now, LastJobID: "job-2222222222222222", + }); err != nil { + return err + } + return st.UpsertCancellation(state.Cancellation{ + ID: "cancel-running-scratch", IdempotencyKey: "cancel-key-running-scratch", Repo: "o/r", IssueNumber: 30, + TriggerCommentID: 911, CancelingUserLogin: "bob", TargetPublicSessionID: "ps-cancel-running", + Status: state.StatusQueued, CreatedAt: now, + }) + }) + fake := &fakeStorage{} + coordinator := &fakeCancelCoordinator{cancelResult: acpx.CancelResult{Confirmed: true, Diagnostics: "cancelled by acpx"}} + dispatcher, _ := newSharedLayoutDispatcher(store, &fakeWorkspaces{}, &fakeCoordinator{}, fake, now) + dispatcher.Acpx = staticAcpxFactory{coordinator: coordinator} + + result, err := dispatcher.DrainCancellations(context.Background(), 0) + if err != nil { + t.Fatalf("DrainCancellations: %v", err) + } + if !result.Executed || result.JobID != "job-2222222222222222" || result.Status != state.StatusCancelled { + t.Fatalf("cancel result = %+v", result) + } + if coordinator.cancelCalls != 1 { + t.Fatalf("acpx cancel calls = %d, want 1", coordinator.cancelCalls) + } + if len(fake.completeCalls) != 1 || fake.completeCalls[0].jobID != "job-2222222222222222" { + t.Fatalf("confirmed cancellation must complete the job scratch: %+v", fake.completeCalls) + } + // Restart-style coordinator rebuild re-records the shared home and the + // session process pool, never a legacy session runtime. + if len(fake.homeCalls) != 1 || len(fake.poolCalls) != 1 || len(fake.recordCalls) != 0 { + t.Fatalf("cancel coordinator rebuild recording: home=%d pool=%d record=%d", + len(fake.homeCalls), len(fake.poolCalls), len(fake.recordCalls)) + } + }) +} + +// TestSharedHomeMirrorsStayAtomicUnderConcurrentRefresh hammers the shared +// runtime home the way concurrent jobs do: two gh mirrors and two codex +// mirrors loop into the same shared directories while a reader continuously +// validates that every observed file is one complete generation, never a +// partial or torn write. +func TestSharedHomeMirrorsStayAtomicUnderConcurrentRefresh(t *testing.T) { + root := t.TempDir() + ghDest := filepath.Join(root, "shared-home", "gh") + codexDest := filepath.Join(root, "shared-home", "codex") + + generation := func(i int) []byte { + header := fmt.Sprintf("%08d", i) + return []byte(header + "\n" + strings.Repeat(header+"|", 500)) + } + writeHostAtomic := func(dir, name string, data []byte) error { + tmp := filepath.Join(dir, ".host-write-"+name) + if err := os.WriteFile(tmp, data, 0o600); err != nil { + return err + } + return os.Rename(tmp, filepath.Join(dir, name)) + } + validate := func(data []byte) error { + header, rest, found := strings.Cut(string(data), "\n") + if !found { + return fmt.Errorf("torn mirror: header without payload (len=%d)", len(data)) + } + if rest != strings.Repeat(header+"|", 500) { + return fmt.Errorf("torn mirror: payload of generation %q is incomplete", header) + } + return nil + } + + errCh := make(chan error, 16) + writersDone := make(chan struct{}) + var wg sync.WaitGroup + for g := 0; g < 2; g++ { + source := filepath.Join(root, fmt.Sprintf("host-gh-%d", g)) + if err := os.MkdirAll(source, 0o700); err != nil { + t.Fatal(err) + } + wg.Add(1) + go func(source string, offset int) { + defer wg.Done() + for i := 0; i < 40; i++ { + if err := writeHostAtomic(source, "hosts.yml", generation(i*2+offset)); err != nil { + errCh <- err + return + } + if err := copyGHConfigDir(source, ghDest); err != nil { + errCh <- fmt.Errorf("gh mirror: %w", err) + return + } + } + }(source, g) + } + for g := 0; g < 2; g++ { + source := filepath.Join(root, fmt.Sprintf("host-codex-%d", g)) + if err := os.MkdirAll(source, 0o700); err != nil { + t.Fatal(err) + } + wg.Add(1) + go func(source string, offset int) { + defer wg.Done() + for i := 0; i < 40; i++ { + if err := writeHostAtomic(source, "auth.json", generation(i*2+offset)); err != nil { + errCh <- err + return + } + if err := copyLimitedCodexConfig(source, codexDest); err != nil { + errCh <- fmt.Errorf("codex mirror: %w", err) + return + } + } + }(source, g) + } + go func() { + wg.Wait() + close(writersDone) + }() + + readerDone := make(chan struct{}) + go func() { + defer close(readerDone) + for { + select { + case <-writersDone: + return + default: + } + for _, path := range []string{filepath.Join(ghDest, "hosts.yml"), filepath.Join(codexDest, "auth.json")} { + data, err := os.ReadFile(path) + if errors.Is(err, os.ErrNotExist) { + continue + } + if err != nil { + errCh <- fmt.Errorf("read %s: %w", path, err) + return + } + if err := validate(data); err != nil { + errCh <- fmt.Errorf("%s: %w", path, err) + return + } + } + } + }() + <-readerDone + + close(errCh) + for err := range errCh { + t.Fatal(err) + } + // The final state must be complete generations of both mirrors. + for _, path := range []string{filepath.Join(ghDest, "hosts.yml"), filepath.Join(codexDest, "auth.json")} { + data, err := os.ReadFile(path) + if err != nil { + t.Fatalf("final mirror %s: %v", path, err) + } + if err := validate(data); err != nil { + t.Fatalf("final mirror %s: %v", path, err) + } + } +} + +// TestSharedHomeMirrorSkipsUnchangedContent proves steady-state refreshes do +// not churn the shared home: a second mirror of byte-identical host config +// performs no write, so the installed file keeps its identity. +func TestSharedHomeMirrorSkipsUnchangedContent(t *testing.T) { + root := t.TempDir() + ghSource := filepath.Join(root, "host-gh") + codexSource := filepath.Join(root, "host-codex") + for _, dir := range []string{ghSource, codexSource} { + if err := os.MkdirAll(dir, 0o700); err != nil { + t.Fatal(err) + } + } + if err := os.WriteFile(filepath.Join(ghSource, "hosts.yml"), []byte("github.com:\n oauth_token: x\n"), 0o600); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(codexSource, "auth.json"), []byte(`{"token":"x"}`), 0o600); err != nil { + t.Fatal(err) + } + ghDest := filepath.Join(root, "shared-home", "gh") + codexDest := filepath.Join(root, "shared-home", "codex") + + mirror := func() { + t.Helper() + if err := copyGHConfigDir(ghSource, ghDest); err != nil { + t.Fatalf("gh mirror: %v", err) + } + if err := copyLimitedCodexConfig(codexSource, codexDest); err != nil { + t.Fatalf("codex mirror: %v", err) + } + } + mirror() + statFile := func(path string) os.FileInfo { + t.Helper() + info, err := os.Stat(path) + if err != nil { + t.Fatalf("stat %s: %v", path, err) + } + return info + } + ghBefore := statFile(filepath.Join(ghDest, "hosts.yml")) + codexBefore := statFile(filepath.Join(codexDest, "auth.json")) + + mirror() + ghAfter := statFile(filepath.Join(ghDest, "hosts.yml")) + codexAfter := statFile(filepath.Join(codexDest, "auth.json")) + if !os.SameFile(ghBefore, ghAfter) || !os.SameFile(codexBefore, codexAfter) { + t.Fatalf("unchanged mirror must not rewrite the shared home files") + } + if !ghAfter.ModTime().Equal(ghBefore.ModTime()) || !codexAfter.ModTime().Equal(codexBefore.ModTime()) { + t.Fatalf("unchanged mirror must not touch modification times") + } + if ghAfter.Mode().Perm() != 0o600 || codexAfter.Mode().Perm() != 0o600 { + t.Fatalf("mirrored files must stay private: gh=%v codex=%v", ghAfter.Mode(), codexAfter.Mode()) + } +} From 4be8a2ed30cca370af1d54575bee09665f99dc05 Mon Sep 17 00:00:00 2001 From: johnlanni Date: Sat, 8 Aug 2026 22:49:00 +0800 Subject: [PATCH 09/17] feat(cli): report runtime usage and evict caches in runner storage reconcile Extend runner storage reconcile with a runtime section: when the sidecar tracks runner-scoped shared homes, the report prints one line per home with protected/cache/unknown bytes from MeasureRuntimeHome plus total job-scratch bytes from MeasureJobScratch, byte counts only. A new --evict-caches flag (requiring --apply) runs ReconcileJobScratch and EvictRuntimeCaches after the main pass and prints the reclaimed bytes; --json wraps the main report with the runtime and eviction sections when either is present. Signed-off-by: johnlanni --- internal/commands/runner_storage.go | 130 ++++++++++++- internal/commands/runner_storage_test.go | 224 +++++++++++++++++++++++ 2 files changed, 350 insertions(+), 4 deletions(-) diff --git a/internal/commands/runner_storage.go b/internal/commands/runner_storage.go index 17f08985..8576e872 100644 --- a/internal/commands/runner_storage.go +++ b/internal/commands/runner_storage.go @@ -5,6 +5,7 @@ import ( "errors" "fmt" "io" + "sort" "strings" "github.com/higress-group/issue-spec/internal/commentrunner" @@ -101,8 +102,9 @@ func (a *app) printRunnerStorageUsage(out io.Writer) { fmt.Fprintln(out, `Usage: issue-spec runner storage reconcile [options] --dry-run|--apply -Reconcile runner-managed storage (.sessions runtimes and .process-workspaces -pools) against current state using the shared classification and deletion +Reconcile runner-managed storage (.sessions runtimes, .process-workspaces +pools, .runner-home shared runtime homes, and .job-scratch directories) +against current state using the shared classification and deletion engine. Performs no issue polling. Defaults resolve through the same runner scope path logic as runner poll. @@ -116,6 +118,8 @@ Options: --storage-orphan-grace orphan observation window (default 168h) --dry-run report classifications and would-delete actions only --apply apply recoverable deletions + --evict-caches with --apply: also reconcile stale job scratch and + evict rebuildable runtime caches, printing reclaimed bytes --json write the JSON report`) } @@ -130,6 +134,7 @@ func (a *app) runRunnerStorageReconcile(ctx context.Context, args []string) int orphanGrace := fs.Duration("storage-orphan-grace", storage.DefaultOrphanGrace, "orphan observation window before unmatched runtime deletion") dryRun := fs.Bool("dry-run", false, "report only, no mutations") apply := fs.Bool("apply", false, "apply recoverable deletions") + evictCaches := fs.Bool("evict-caches", false, "with --apply: reconcile stale job scratch and evict rebuildable runtime caches") jsonOut := fs.Bool("json", false, "write JSON output") fs.Var(&repoValues, "repo", "repository owner/name for scoped defaults; repeat or comma-separate") if argsContainHelp(args) { @@ -144,6 +149,10 @@ func (a *app) runRunnerStorageReconcile(ctx context.Context, args []string) int a.errorf("runner storage reconcile requires exactly one of --dry-run or --apply\n") return 2 } + if *evictCaches && !*apply { + a.errorf("runner storage reconcile --evict-caches requires --apply\n") + return 2 + } if *orphanGrace < 0 { a.errorf("--storage-orphan-grace must not be negative\n") return 2 @@ -246,17 +255,109 @@ func (a *app) runRunnerStorageReconcile(ctx context.Context, args []string) int a.errorf("runner storage reconcile: %v\n", err) return 1 } + var eviction *runtimeEvictionReport + if *evictCaches { + eviction = &runtimeEvictionReport{} + scratchReport, err := service.ReconcileJobScratch(ctx, true) + if err != nil { + a.errorf("runner storage reconcile job scratch: %v\n", err) + return 1 + } + eviction.Scratch = scratchReport + cacheReport, err := service.EvictRuntimeCaches(ctx, true) + if err != nil { + a.errorf("runner storage reconcile cache eviction: %v\n", err) + return 1 + } + eviction.Caches = cacheReport + } + runtimeUsage := collectRuntimeUsage(service) if *jsonOut { + if runtimeUsage != nil || eviction != nil { + return a.outputJSON(runnerStorageReconcileOutput{Report: report, Runtime: runtimeUsage, Eviction: eviction}) + } return a.outputJSON(report) } - printStorageReport(a.out, report) + printStorageReport(a.out, report, runtimeUsage, eviction) if report.ReportOnly { return 1 } return 0 } -func printStorageReport(out io.Writer, report storage.Report) { +// runnerStorageReconcileOutput is the JSON shape when the runtime usage +// section or the --evict-caches passes accompany the main reconcile report. +type runnerStorageReconcileOutput struct { + Report storage.Report `json:"report"` + Runtime *runtimeUsageSection `json:"runtime,omitempty"` + Eviction *runtimeEvictionReport `json:"eviction,omitempty"` +} + +// runtimeUsageSection is the measured runtime view: one line per recorded +// runner-scoped shared home plus total job-scratch bytes. It carries byte +// counts only, never file contents. +type runtimeUsageSection struct { + Homes []runtimeHomeUsageLine `json:"homes,omitempty"` + ScratchBytes int64 `json:"scratch_bytes"` + Diagnostics []string `json:"diagnostics,omitempty"` +} + +type runtimeHomeUsageLine struct { + Hash string `json:"hash"` + Repo string `json:"repo"` + ProtectedBytes int64 `json:"protected_bytes"` + CacheBytes int64 `json:"cache_bytes"` + UnknownBytes int64 `json:"unknown_bytes"` +} + +// runtimeEvictionReport carries the --evict-caches pass results: stale job +// scratch reconciliation plus runtime cache eviction. +type runtimeEvictionReport struct { + Scratch storage.RuntimeReconcileReport `json:"job_scratch"` + Caches storage.RuntimeReconcileReport `json:"caches"` +} + +// collectRuntimeUsage measures every recorded runner home and the job scratch +// base. It returns nil when no runner home records exist, so legacy-layout +// roots keep the previous report shape. Measurement failures degrade to +// diagnostics: the reconcile pass already succeeded and its report stays +// valid. +func collectRuntimeUsage(service *storage.Service) *runtimeUsageSection { + records := make([]storage.PhysicalResource, 0) + for _, resource := range service.Store().State().Resources { + if resource.Kind == storage.ResourceKindRunnerHome { + records = append(records, resource) + } + } + if len(records) == 0 { + return nil + } + sort.Slice(records, func(i, j int) bool { return records[i].ID < records[j].ID }) + section := &runtimeUsageSection{} + for _, record := range records { + usage, err := storage.MeasureRuntimeHome(record.Path) + if err != nil { + section.Diagnostics = append(section.Diagnostics, fmt.Sprintf("runtime home %s measurement: %v", record.PhysicalHash, err)) + continue + } + section.Homes = append(section.Homes, runtimeHomeUsageLine{ + Hash: record.PhysicalHash, + Repo: record.Repo, + ProtectedBytes: usage.ProtectedBytes, + CacheBytes: usage.CacheBytes, + UnknownBytes: usage.UnknownBytes, + }) + } + scratch, err := storage.MeasureJobScratch(service.Root()) + if err != nil { + section.Diagnostics = append(section.Diagnostics, fmt.Sprintf("job scratch measurement: %v", err)) + } else { + section.ScratchBytes = scratch + } + return section +} + +func printStorageReport(out io.Writer, report storage.Report, runtimeUsage *runtimeUsageSection, eviction *runtimeEvictionReport) { mode := "apply" if report.DryRun { mode = "dry-run" @@ -284,4 +385,25 @@ func printStorageReport(out io.Writer, report storage.Report) { for _, diagnostic := range report.Diagnostics { fmt.Fprintf(out, "diagnostic: %s\n", diagnostic) } + if runtimeUsage != nil { + fmt.Fprintf(out, "runtime: runner_homes=%d job_scratch=%d bytes\n", len(runtimeUsage.Homes), runtimeUsage.ScratchBytes) + for _, home := range runtimeUsage.Homes { + fmt.Fprintf(out, "- runner_home %s repo=%s protected=%d cache=%d unknown=%d\n", + home.Hash, home.Repo, home.ProtectedBytes, home.CacheBytes, home.UnknownBytes) + } + for _, diagnostic := range runtimeUsage.Diagnostics { + fmt.Fprintf(out, "diagnostic: %s\n", diagnostic) + } + } + if eviction != nil { + reclaimed := eviction.Scratch.ReclaimedBytes + eviction.Caches.ReclaimedBytes + fmt.Fprintf(out, "evict-caches: reclaimed=%d bytes scratch_removed=%d caches_evicted=%d\n", + reclaimed, len(eviction.Scratch.ScratchRemoved), len(eviction.Caches.CacheEvicted)) + for _, diagnostic := range eviction.Scratch.Diagnostics { + fmt.Fprintf(out, "diagnostic: %s\n", diagnostic) + } + for _, diagnostic := range eviction.Caches.Diagnostics { + fmt.Fprintf(out, "diagnostic: %s\n", diagnostic) + } + } } diff --git a/internal/commands/runner_storage_test.go b/internal/commands/runner_storage_test.go index d3237db9..ef598101 100644 --- a/internal/commands/runner_storage_test.go +++ b/internal/commands/runner_storage_test.go @@ -6,6 +6,7 @@ import ( "encoding/json" "os" "path/filepath" + "strconv" "strings" "testing" "time" @@ -449,3 +450,226 @@ func TestRunnerStorageLifecycleFailsClosed(t *testing.T) { t.Fatal("reconcile must surface the construction error") } } + +// seedSharedRuntimeFixture builds a fresh runner root on the shared layout: +// one prepared runner-scoped runtime home recorded in the storage sidecar. +func seedSharedRuntimeFixture(t *testing.T) (statePath, workspaceRoot string, scope storage.RuntimeScope, home storage.RuntimeHomePaths) { + t.Helper() + base := t.TempDir() + statePath = filepath.Join(base, "state.json") + workspaceRoot = filepath.Join(base, "workspaces") + if err := os.MkdirAll(workspaceRoot, 0o700); err != nil { + t.Fatalf("mkdir root: %v", err) + } + if err := crstate.SaveFile(statePath, crstate.NewState()); err != nil { + t.Fatalf("save state: %v", err) + } + scope = storage.RuntimeScope{Hostname: "github.com", Repo: "o/r", Runner: "bot"} + home, err := storage.PrepareRuntimeHome(workspaceRoot, scope) + if err != nil { + t.Fatalf("prepare runtime home: %v", err) + } + svc, err := storage.NewService(storage.ServiceConfig{ + WorkspaceRoot: workspaceRoot, + StateLoader: func(context.Context) (crstate.RunnerState, error) { return crstate.NewState(), nil }, + }) + if err != nil { + t.Fatalf("NewService: %v", err) + } + defer svc.Close() + if err := svc.RecordRuntimeHome(context.Background(), scope, home); err != nil { + t.Fatalf("record runtime home: %v", err) + } + return statePath, workspaceRoot, scope, home +} + +func writeSizedFile(t *testing.T, path string, size int) { + t.Helper() + if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path, bytes.Repeat([]byte("x"), size), 0o600); err != nil { + t.Fatal(err) + } +} + +func TestRunnerStorageReconcileReportIncludesRuntimeSection(t *testing.T) { + statePath, workspaceRoot, scope, home := seedSharedRuntimeFixture(t) + writeSizedFile(t, filepath.Join(home.Home, ".gitconfig"), 100) + writeSizedFile(t, filepath.Join(home.Home, ".cache", "blob"), 200) + writeSizedFile(t, filepath.Join(home.Home, "misc.bin"), 50) + writeSizedFile(t, filepath.Join(workspaceRoot, ".job-scratch", "job-aaaaaaaaaaaaaaaa", "tmp", "f"), 30) + + usage, err := storage.MeasureRuntimeHome(home.Root) + if err != nil { + t.Fatal(err) + } + var out, errOut bytes.Buffer + app := newApp(strings.NewReader(""), &out, &errOut) + code := app.runRunner(context.Background(), []string{"storage", "reconcile", + "--state", statePath, "--workspace-root", workspaceRoot, "--dry-run"}) + if code != 0 { + t.Fatalf("exit=%d stderr=%q", code, errOut.String()) + } + text := out.String() + hash, err := storage.RuntimeScopeHash(scope) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(text, "runtime: runner_homes=1 job_scratch=30 bytes") { + t.Fatalf("runtime section header missing or wrong:\n%s", text) + } + wantHome := "- runner_home " + hash + " repo=o/r protected=" + + strconv.FormatInt(usage.ProtectedBytes, 10) + " cache=" + + strconv.FormatInt(usage.CacheBytes, 10) + " unknown=" + + strconv.FormatInt(usage.UnknownBytes, 10) + if !strings.Contains(text, wantHome) { + t.Fatalf("runtime home line %q missing:\n%s", wantHome, text) + } + if usage.CacheBytes != 200 || usage.UnknownBytes != 50 { + t.Fatalf("measurement sanity: cache=%d unknown=%d, want 200/50", usage.CacheBytes, usage.UnknownBytes) + } + // A dry run measures but never deletes. + if _, err := os.Lstat(filepath.Join(home.Home, ".cache", "blob")); err != nil { + t.Fatalf("dry-run must not delete cache content: %v", err) + } +} + +func TestRunnerStorageReconcileWithoutRunnerHomesOmitsRuntimeSection(t *testing.T) { + statePath, workspaceRoot, _ := seedStorageFixture(t) + var out, errOut bytes.Buffer + app := newApp(strings.NewReader(""), &out, &errOut) + code := app.runRunner(context.Background(), []string{"storage", "reconcile", + "--state", statePath, "--workspace-root", workspaceRoot, "--dry-run"}) + if code != 0 { + t.Fatalf("exit=%d stderr=%q", code, errOut.String()) + } + if strings.Contains(out.String(), "runtime: runner_homes=") { + t.Fatalf("legacy root must not print a runtime section:\n%s", out.String()) + } +} + +func TestRunnerStorageReconcileEvictCachesRequiresApply(t *testing.T) { + statePath, workspaceRoot, _, _ := seedSharedRuntimeFixture(t) + var out, errOut bytes.Buffer + app := newApp(strings.NewReader(""), &out, &errOut) + code := app.runRunner(context.Background(), []string{"storage", "reconcile", + "--state", statePath, "--workspace-root", workspaceRoot, "--dry-run", "--evict-caches"}) + if code != 2 { + t.Fatalf("exit=%d, want 2 for --evict-caches without --apply (stdout=%q)", code, out.String()) + } +} + +func TestRunnerStorageReconcileEvictCachesEvictsOnlyCaches(t *testing.T) { + statePath, workspaceRoot, _, home := seedSharedRuntimeFixture(t) + writeSizedFile(t, filepath.Join(home.Home, ".cache", "blob"), 400) + writeSizedFile(t, filepath.Join(home.Home, "go", "pkg", "mod", "mod.bin"), 300) + writeSizedFile(t, filepath.Join(home.Home, ".gitconfig"), 100) + writeSizedFile(t, filepath.Join(home.CodexHome, "auth.json"), 40) + writeSizedFile(t, filepath.Join(workspaceRoot, ".job-scratch", "job-bbbbbbbbbbbbbbbb", "tmp", "f"), 60) + + var out, errOut bytes.Buffer + app := newApp(strings.NewReader(""), &out, &errOut) + code := app.runRunner(context.Background(), []string{"storage", "reconcile", + "--state", statePath, "--workspace-root", workspaceRoot, "--apply", "--evict-caches"}) + if code != 0 { + t.Fatalf("exit=%d stderr=%q", code, errOut.String()) + } + text := out.String() + if !strings.Contains(text, "evict-caches: reclaimed=760 bytes scratch_removed=1 caches_evicted=2") { + t.Fatalf("eviction summary missing or wrong:\n%s", text) + } + for _, gone := range []string{ + filepath.Join(home.Home, ".cache"), + filepath.Join(home.Home, "go", "pkg", "mod"), + filepath.Join(workspaceRoot, ".job-scratch", "job-bbbbbbbbbbbbbbbb"), + } { + if _, err := os.Lstat(gone); !os.IsNotExist(err) { + t.Fatalf("eviction must remove %s, err=%v", gone, err) + } + } + for _, kept := range []string{ + filepath.Join(home.Home, ".gitconfig"), + filepath.Join(home.Root, "scope.json"), + filepath.Join(home.CodexHome, "auth.json"), + } { + if _, err := os.Lstat(kept); err != nil { + t.Fatalf("eviction must keep protected path %s: %v", kept, err) + } + } + + // The JSON shape wraps the main report with the runtime and eviction + // sections once --evict-caches runs. + writeSizedFile(t, filepath.Join(home.Home, ".cache", "blob2"), 10) + var jsonOutBuf, jsonErrBuf bytes.Buffer + jsonApp := newApp(strings.NewReader(""), &jsonOutBuf, &jsonErrBuf) + code = jsonApp.runRunner(context.Background(), []string{"storage", "reconcile", + "--state", statePath, "--workspace-root", workspaceRoot, "--apply", "--evict-caches", "--json"}) + if code != 0 { + t.Fatalf("json exit=%d stderr=%q", code, jsonErrBuf.String()) + } + var decoded struct { + Report storage.Report `json:"report"` + Runtime *struct { + Homes []struct { + Hash string `json:"hash"` + } `json:"homes"` + } `json:"runtime"` + Eviction *struct { + Caches storage.RuntimeReconcileReport `json:"caches"` + } `json:"eviction"` + } + if err := json.Unmarshal(jsonOutBuf.Bytes(), &decoded); err != nil { + t.Fatalf("decode wrapped report: %v\n%s", err, jsonOutBuf.String()) + } + if decoded.Runtime == nil || len(decoded.Runtime.Homes) != 1 || decoded.Eviction == nil || len(decoded.Eviction.Caches.CacheEvicted) != 1 { + t.Fatalf("wrapped JSON missing runtime/eviction sections: %s", jsonOutBuf.String()) + } +} + +func TestRunnerStorageReconcileApplyRecoversRecordedStaleScratch(t *testing.T) { + statePath, workspaceRoot, _, _ := seedSharedRuntimeFixture(t) + scratch, err := storage.PrepareJobScratch(workspaceRoot, "job-cccccccccccccccc") + if err != nil { + t.Fatal(err) + } + writeSizedFile(t, filepath.Join(scratch.Tmp, "f"), 25) + svc, err := storage.NewService(storage.ServiceConfig{ + WorkspaceRoot: workspaceRoot, + StateLoader: func(context.Context) (crstate.RunnerState, error) { return crstate.NewState(), nil }, + }) + if err != nil { + t.Fatalf("NewService: %v", err) + } + if err := svc.RecordJobScratch(context.Background(), "o/r", "job-cccccccccccccccc", scratch.Root); err != nil { + t.Fatalf("record job scratch: %v", err) + } + svc.Close() + + var out, errOut bytes.Buffer + app := newApp(strings.NewReader(""), &out, &errOut) + code := app.runRunner(context.Background(), []string{"storage", "reconcile", + "--state", statePath, "--workspace-root", workspaceRoot, "--apply", "--evict-caches"}) + if code != 0 { + t.Fatalf("exit=%d stderr=%q", code, errOut.String()) + } + if !strings.Contains(out.String(), "evict-caches: reclaimed=25 bytes scratch_removed=1 caches_evicted=0") { + t.Fatalf("stale scratch recovery summary missing:\n%s", out.String()) + } + if _, err := os.Lstat(scratch.Root); !os.IsNotExist(err) { + t.Fatalf("stale recorded scratch must be removed, err=%v", err) + } + svc2, err := storage.NewService(storage.ServiceConfig{ + WorkspaceRoot: workspaceRoot, + StateLoader: func(context.Context) (crstate.RunnerState, error) { return crstate.NewState(), nil }, + }) + if err != nil { + t.Fatalf("NewService reopen: %v", err) + } + defer svc2.Close() + for id, resource := range svc2.Store().State().Resources { + if resource.Kind == storage.ResourceKindJobScratch { + t.Fatalf("stale scratch record %s must be garbage-collected", id) + } + } +} From 12add09d563a9ed0023a7ea0266081d1df232864 Mon Sep 17 00:00:00 2001 From: johnlanni Date: Sat, 8 Aug 2026 22:54:12 +0800 Subject: [PATCH 10/17] docs(runner): document the shared runtime layout and breaking cutover Document the runner-scoped shared HOME (.runner-home//) and per-job scratch (.job-scratch//), the storage accounting categories and --evict-caches reclamation, and the explicit breaking cutover: a fresh runner root is required, pre-cutover sessions are not resumable, and the new binary never reads for import, modifies, or deletes an old root. Applied to both runner guides and their zh-CN mirrors. Signed-off-by: johnlanni --- docs/runner.md | 21 +++++++++++-- docs/runner.zh-CN.md | 21 +++++++++++-- docs/self-hosting/runner.md | 51 +++++++++++++++++++++++++------ docs/self-hosting/runner.zh-CN.md | 41 +++++++++++++++++++------ 4 files changed, 110 insertions(+), 24 deletions(-) diff --git a/docs/runner.md b/docs/runner.md index 68e17cf2..2d0b22b6 100644 --- a/docs/runner.md +++ b/docs/runner.md @@ -53,7 +53,7 @@ is `1.1.2`: Save this as `~/.acpx/config.json` for the Runner service user and keep the version under operator configuration management. The runner copies only the -selected Codex override into each job's isolated ACPX home, so this pin applies +selected Codex override into the runner-scoped shared ACPX home, so this pin applies inside bubblewrap without copying unrelated ACPX configuration. Hosts without npm registry access should pre-cache the exact pinned package, for example `npm cache add @agentclientprotocol/codex-acp@1.1.2`. @@ -169,7 +169,7 @@ Useful runner options: - `--gh-config-dir ` selects the host GitHub CLI config directory mirrored into the sandbox. By default the runner derives it from the host GitHub CLI environment. - `--allow-cancel=false` disables `/cancel` intake. -Runner storage is anchored at the workspace root. The runner keeps a private `.storage` directory beside the managed clones: a sidecar inventory of session runtime roots (`.sessions/`) and PROCESS workspace pools (`.process-workspaces/`), an owner lock, and first-migration backups. One canonical root has exactly one destructive owner: `runner poll` and `runner serve` hold the owner lock for the process lifetime, and a second runner started against the same root fails with "stop the old runner before starting a new one". Always stop the old runner before starting a new one against the same state and workspace pair. `runner storage reconcile` also acquires the owner lock in both modes — `--dry-run` mutates nothing but is still exclusive with a live runner; only `runner preflight` is lock-free. +Runner storage is anchored at the workspace root. The runner keeps a private `.storage` directory beside the managed clones: a sidecar inventory of every managed physical resource — runner-scoped shared runtime homes (`.runner-home//`), per-job scratch directories (`.job-scratch//`), and PROCESS workspace pools (`.process-workspaces//`), plus per-session runtime roots (`.sessions//`) on roots created by older binaries — an owner lock, and first-migration backups. One canonical root has exactly one destructive owner: `runner poll` and `runner serve` hold the owner lock for the process lifetime, and a second runner started against the same root fails with "stop the old runner before starting a new one". Always stop the old runner before starting a new one against the same state and workspace pair. `runner storage reconcile` also acquires the owner lock in both modes — `--dry-run` mutates nothing but is still exclusive with a live runner; only `runner preflight` is lock-free. Inspect or reclaim storage explicitly through the same engine the runner uses internally: @@ -178,10 +178,25 @@ issue-spec runner storage reconcile --state --workspace-root issue-spec runner storage reconcile --state --workspace-root --apply ``` -`--dry-run` classifies every inventoried resource (`protected`, `retired_known`, `orphan_observed`, `rejected`) and reports `would_delete` actions without mutating anything. `--apply` deletes only eligible resources, and re-validates each one against freshly reloaded runner state immediately before removal. Before the first destructive pass on a root, the raw pre-migration runner state is preserved under `.storage/backups/state-first.json`; when that backup cannot be written, deletions are skipped for the pass. The command exits non-zero when the sidecar is report-only: a sidecar bound to a different canonical root identity, or written by a newer schema version, is inventoried read-only and is never mutated or deleted. +`--dry-run` classifies every inventoried resource (`protected`, `retired_known`, `orphan_observed`, `rejected`) and reports `would_delete` actions without mutating anything. `--apply` deletes only eligible resources, and re-validates each one against freshly reloaded runner state immediately before removal. Before the first destructive pass on a root, the raw pre-migration runner state is preserved under `.storage/backups/state-first.json`; when that backup cannot be written, deletions are skipped for the pass. The command exits non-zero when the sidecar is report-only: a sidecar bound to a different canonical root identity, or written by a newer schema version, is inventoried read-only and is never mutated or deleted. When shared runtime homes exist, the report adds a `runtime:` section with one line per home — protected identity/configuration bytes, rebuildable cache bytes, and unknown bytes — plus total job-scratch bytes. `--apply --evict-caches` additionally reconciles stale job scratch and evicts only the rebuildable cache directories of each recorded home, printing the reclaimed bytes. PROCESS pool deletion is deliberately conservative. A retired pool is removed only when its owning clone is present and inspection proves the pool empty: no active leases, ownership markers, registered worktrees, or stray files. Uncertain pools are preserved with an operator remediation diagnostic and are never force-abandoned; inspect or recover them through the owning PROCESS (for example `issue-spec workflow workspace reconcile`) before removing anything manually. Runtime and pool deletion failures defer the owning workspace from grouped workspace cleanup for that pass so remediation evidence is not destroyed. +Every dispatched job runs with two runner-owned anchors below the workspace root: + +- `.runner-home//` is the persistent runtime HOME shared by all jobs of one runtime scope (hostname, issue-backend profile realm, repository, and runner identity). It holds the agent-visible `home/`, `gh/`, `xdg/`, `codex/`, and `acpx-runtime/` directories plus a `scope.json` binding that fails closed if a different scope ever resolves to the same path. Sharing one HOME keeps package and build caches (the Go module and build caches, npm caches) warm across jobs of the same scope instead of rebuilding them per session, while host credential mirrors are refreshed atomically per dispatch so concurrent jobs never observe partial files. +- `.job-scratch//` is the job's disposable scratch: `TMPDIR`, `GOTMPDIR`, `XDG_DATA_HOME`, and `XDG_STATE_HOME` bind to fixed paths below `/tmp/issue-spec-scratch` inside bubblewrap (explicit unsafe mode exports the host paths directly). Scratch is removed when the job reaches a terminal state — completion, failure, or cancellation — and stale scratch is reclaimed by `runner storage reconcile --apply --evict-caches`. + +Storage accounting treats a home's agent identity and configuration (including `~/.ssh`, `~/.gitconfig`, and the mirrored `gh`/`codex` configuration) as protected, never as eviction targets; only rebuildable caches (`~/.cache`, `~/.npm`, `~/go/pkg/mod`) are evicted, most rebuildable first. + +Upgrading to the shared runtime layout is a breaking cutover, not an in-place upgrade. A binary with the shared layout never reads for import, modifies, or deletes a root created by an older binary, and sessions created before the cutover are not resumable afterwards — their acpx records live in the old root's per-session runtimes, which the new binary does not consult. Cut over with a fresh runner root: + +1. Stop the old runner and let in-flight jobs finish or cancel them. +2. Optionally archive the old root (state file, `workspaces/`, and `.storage/`) for audit. +3. Start the new binary against a fresh, empty `--state`/`--workspace-root` pair — either new paths or the same paths after the old root has been moved away. +4. Verify with `runner preflight` and one `/new` command; pre-cutover sessions do not carry over, so follow-ups start new sessions. +5. Archive or delete the old root separately once the new root is proven. + On Linux, runner dispatch uses bubblewrap by default to keep coordinator filesystem writes inside the managed session clone and that session's PROCESS workspace pool while still allowing network access for GitHub, model, and package operations. Native children share that outer boundary; bubblewrap does not create a separate sandbox per child. Install bubblewrap or set `ISSUE_SPEC_BWRAP_PATH` / `--bwrap-path` when it is not on `PATH`. If bubblewrap is unavailable or unsupported, the runner fails preflight instead of silently running without isolation. Use `--unsafe-no-sandbox` only as an explicit operator choice, including on macOS where bubblewrap is unavailable. There is no automatic fallback from sandboxed mode: diff --git a/docs/runner.zh-CN.md b/docs/runner.zh-CN.md index 9adff903..2a51164e 100644 --- a/docs/runner.zh-CN.md +++ b/docs/runner.zh-CN.md @@ -49,7 +49,7 @@ fallback，而不是运维侧的版本策略：即使主机上的 `codex` CLI ``` 将该内容保存为 Runner 服务用户的 `~/.acpx/config.json`，并将版本纳入运维配置管理。 -runner 只会把选中的 Codex override 复制到每个任务隔离的 ACPX home，因此该固定版本会在 +runner 只会把选中的 Codex override 复制到 runner 作用域共享的 ACPX home，因此该固定版本会在 bubblewrap 内生效，而不会携带其他无关的 ACPX 配置。无法访问 npm registry 的主机应预先 缓存精确的固定版本，例如 `npm cache add @agentclientprotocol/codex-acp@1.1.2`。 @@ -156,7 +156,7 @@ issue-spec runner poll \ - `--gh-config-dir ` 选择要镜像进沙箱的宿主 GitHub CLI 配置目录。默认情况下 runner 会从宿主 GitHub CLI 环境推导。 - `--allow-cancel=false` 关闭 `/cancel` intake。 -Runner 的存储以 workspace root 为锚点。runner 在受管克隆旁边维护一个私有的 `.storage` 目录：session runtime 根（`.sessions/`）与 PROCESS workspace pool（`.process-workspaces/`）的 sidecar 清单、owner 锁，以及首次迁移备份。一个规范化 root 只有一个具备破坏性的 owner：`runner poll` 与 `runner serve` 在进程生命周期内持有 owner 锁，在同一 root 上启动第二个 runner 会失败并提示「stop the old runner before starting a new one」。在同一 state 与 workspace 对上启动新 runner 之前，务必先停止旧 runner。`runner storage reconcile` 在两种模式下也会获取 owner 锁——`--dry-run` 不做任何修改，但仍与存活 runner 互斥；只有 `runner preflight` 不加锁。 +Runner 的存储以 workspace root 为锚点。runner 在受管克隆旁边维护一个私有的 `.storage` 目录：覆盖所有受管物理资源的 sidecar 清单——runner 作用域共享 runtime home（`.runner-home//`）、按 job 划分的 scratch 目录（`.job-scratch//`）、PROCESS workspace pool（`.process-workspaces//`），以及旧版本二进制创建的 root 上的 per-session runtime 根（`.sessions//`）——以及 owner 锁与首次迁移备份。一个规范化 root 只有一个具备破坏性的 owner：`runner poll` 与 `runner serve` 在进程生命周期内持有 owner 锁，在同一 root 上启动第二个 runner 会失败并提示「stop the old runner before starting a new one」。在同一 state 与 workspace 对上启动新 runner 之前，务必先停止旧 runner。`runner storage reconcile` 在两种模式下也会获取 owner 锁——`--dry-run` 不做任何修改，但仍与存活 runner 互斥；只有 `runner preflight` 不加锁。 可以通过 runner 内部使用的同一引擎显式检查或回收存储： @@ -165,10 +165,25 @@ issue-spec runner storage reconcile --state --workspace-root issue-spec runner storage reconcile --state --workspace-root --apply ``` -`--dry-run` 对清单中的每个资源分类（`protected`、`retired_known`、`orphan_observed`、`rejected`）并报告 `would_delete` 动作，不做任何修改。`--apply` 只删除符合条件的资源，并在删除前立即对照重新加载的 runner 状态重新校验每一项。在对某个 root 执行首次破坏性 pass 之前，原始的迁移前 runner 状态会被保存到 `.storage/backups/state-first.json`；如果该备份无法写入，本次 pass 会跳过删除。当 sidecar 处于 report-only 状态时命令以非零码退出：绑定到不同规范化 root 身份、或由更新 schema 版本写入的 sidecar 只会被只读清点，绝不会被修改或删除。 +`--dry-run` 对清单中的每个资源分类（`protected`、`retired_known`、`orphan_observed`、`rejected`）并报告 `would_delete` 动作，不做任何修改。`--apply` 只删除符合条件的资源，并在删除前立即对照重新加载的 runner 状态重新校验每一项。在对某个 root 执行首次破坏性 pass 之前，原始的迁移前 runner 状态会被保存到 `.storage/backups/state-first.json`；如果该备份无法写入，本次 pass 会跳过删除。当 sidecar 处于 report-only 状态时命令以非零码退出：绑定到不同规范化 root 身份、或由更新 schema 版本写入的 sidecar 只会被只读清点，绝不会被修改或删除。当存在共享 runtime home 时，报告会增加 `runtime:` 小节：每个 home 一行——受保护的 identity/配置字节数、可重建 cache 字节数与未知字节数——以及 job scratch 总字节数。`--apply --evict-caches` 还会 reconcile 过期 job scratch，并只驱逐每个已记录 home 中可重建的 cache 目录，打印回收的字节数。 PROCESS pool 的删除刻意保持保守。仅当所属 clone 存在且 inspection 证明 pool 为空时才会删除已退役的 pool：没有活跃 lease、ownership marker、已注册 worktree 或残留文件。不确定的 pool 会被保留并给出运维处置诊断，绝不会被强制放弃；请通过所属 PROCESS（例如 `issue-spec workflow workspace reconcile`）检查或恢复后，再手动移除任何内容。runtime 与 pool 的删除失败会把所属 workspace 从本次分组 workspace 清理中 defer，以免破坏处置证据。 +每个被分发的 job 都使用 workspace root 下两个由 runner 拥有的锚点： + +- `.runner-home//` 是同一 runtime 作用域（hostname、issue 后端 profile realm、仓库与 runner 身份）下所有 job 共享的持久 runtime HOME。它包含 agent 可见的 `home/`、`gh/`、`xdg/`、`codex/`、`acpx-runtime/` 目录，以及一个 `scope.json` 绑定；若不同的作用域解析到同一路径，绑定会 fail-closed。共享同一个 HOME 可以让包与构建缓存（Go module 与构建缓存、npm 缓存）在同一作用域的后续 job 中保持温热，而不是每个 session 重建；宿主凭据镜像在每次分发时原子刷新，并发 job 绝不会读到写了一半的文件。 +- `.job-scratch//` 是该 job 的一次性 scratch：`TMPDIR`、`GOTMPDIR`、`XDG_DATA_HOME` 与 `XDG_STATE_HOME` 在 bubblewrap 内绑定到 `/tmp/issue-spec-scratch` 下的固定路径（显式 unsafe 模式直接导出宿主路径）。scratch 在 job 进入终态——完成、失败或取消——时被移除；残留的 scratch 由 `runner storage reconcile --apply --evict-caches` 回收。 + +存储记账把 home 中的 agent identity 与配置（包括 `~/.ssh`、`~/.gitconfig` 以及镜像来的 `gh`/`codex` 配置）视为受保护对象，绝不作为驱逐目标；只有可重建的 cache（`~/.cache`、`~/.npm`、`~/go/pkg/mod`）会被驱逐，且按最容易重建的优先。 + +升级到共享 runtime 布局是一次破坏性切换，不是原地升级。带有共享布局的二进制绝不会读取导入、修改或删除旧版本二进制创建的 root；切换前创建的 session 在切换后不可恢复——它们的 acpx 记录保存在旧 root 的 per-session runtime 中，新二进制不会查阅。请使用全新的 runner root 完成切换： + +1. 停止旧 runner，让在途 job 跑完或先取消它们。 +2. 如需审计，可先行归档旧 root（state 文件、`workspaces/` 与 `.storage/`）。 +3. 以全新的空 `--state`/`--workspace-root` 对启动新二进制——要么换新路径，要么先把旧 root 移走再用原路径。 +4. 用 `runner preflight` 和一条 `/new` 命令验证；切换前的 session 不会迁移，后续对话请开新 session。 +5. 新 root 验证通过后，再单独归档或删除旧 root。 + 在 Linux 上，runner 分发默认使用 bubblewrap，把 coordinator 的文件系统写入限制在受管 session clone 与该 session 的 PROCESS workspace pool 内，同时仍允许 GitHub、model 与包操作的网络访问。原生 child 共享这一外层边界；bubblewrap 不会为每个 child 创建独立 sandbox。当 bubblewrap 不在 `PATH` 上时，请安装它或设置 `ISSUE_SPEC_BWRAP_PATH` / `--bwrap-path`。若 bubblewrap 不可用或不受支持，runner 会让 preflight 失败，而不是在没有隔离的情况下静默运行。 只有作为显式的运维选择时才使用 `--unsafe-no-sandbox`；macOS 上没有 bubblewrap 时也必须显式指定，不存在从 sandbox 模式自动降级： diff --git a/docs/self-hosting/runner.md b/docs/self-hosting/runner.md index a210d4eb..1955830b 100644 --- a/docs/self-hosting/runner.md +++ b/docs/self-hosting/runner.md @@ -440,13 +440,27 @@ rerun `runner preflight --verify-agent-runtime` as the same user. ### Storage lifecycle and single owner The runner anchors storage at `--workspace-root`: a private `.storage` -directory holds the sidecar inventory of session runtime roots and PROCESS -workspace pools, an owner lock, and first-migration backups. One canonical +directory holds the sidecar inventory of every managed physical resource — +runner-scoped shared runtime homes (`.runner-home//`), per-job +scratch directories (`.job-scratch//`), and PROCESS workspace pools, +plus per-session runtime roots (`.sessions//`) on roots created by +older binaries — an owner lock, and first-migration backups. One canonical root has exactly one destructive owner, so run exactly one systemd unit per state/workspace pair. A second `runner serve` (or `runner poll`) against the same root fails at startup with "stop the old runner before starting a new one"; always stop the old unit before starting a replacement. +Every dispatched job shares the persistent runtime HOME of its runtime scope +(hostname, profile realm, repository, and runner identity) at +`.runner-home//` — agent `home/`, `gh/`, `xdg/`, `codex/`, and +`acpx-runtime/` directories plus a `scope.json` binding that fails closed on +a foreign scope — so package and build caches warm later jobs of the same +scope. Each job additionally receives a disposable `.job-scratch//` +(`TMPDIR`, `GOTMPDIR`, `XDG_DATA_HOME`, `XDG_STATE_HOME`) that is removed +when the job reaches a terminal state. Agent identity and configuration in a +home are protected; only rebuildable caches (`~/.cache`, `~/.npm`, +`~/go/pkg/mod`) are eviction-eligible. + Two optional flags tune storage behavior on `runner serve`: - `--storage-min-free-bytes ` delays dispatch when the workspace @@ -468,13 +482,32 @@ sudo -u issue-spec-runner issue-spec --profile team runner storage reconcile \ `--dry-run` only reports classifications and `would_delete` actions; `--apply` deletes eligible resources after re-validating each against freshly -reloaded runner state. Both modes acquire the owner lock, so run the command -while the unit is stopped. If the command reports the sidecar as report-only -(different root identity or newer schema), it inventories read-only and exits -non-zero; do not delete `.storage` by hand unless you also accept losing -ownership and orphan-observation proof. Retired PROCESS pools are deleted -only when proven empty; uncertain pools are preserved with a remediation -diagnostic and are never force-abandoned. +reloaded runner state. When shared runtime homes exist, the report adds a +`runtime:` section with per-home protected/cache/unknown bytes and total +job-scratch bytes, and `--apply --evict-caches` also reconciles stale job +scratch and evicts the rebuildable cache directories of each recorded home, +printing the reclaimed bytes. Both modes acquire the owner lock, so run the +command while the unit is stopped. If the command reports the sidecar as +report-only (different root identity or newer schema), it inventories +read-only and exits non-zero; do not delete `.storage` by hand unless you +also accept losing ownership and orphan-observation proof. Retired PROCESS +pools are deleted only when proven empty; uncertain pools are preserved with +a remediation diagnostic and are never force-abandoned. + +Upgrading a unit to the shared runtime layout is a breaking cutover, not an +in-place upgrade: the new binary never reads for import, modifies, or deletes +an old root, and pre-cutover sessions are not resumable because their acpx +records live in the old root's per-session runtimes. Cut over with a fresh +runner root: + +1. Stop the old unit (`systemctl stop issue-spec-runner`) and let in-flight + jobs finish or cancel them first. +2. Optionally archive the old state/workspace root for audit. +3. Point the unit at a fresh, empty `--state`/`--workspace-root` pair (or + move the old root away first) and start the new binary. +4. Verify with `runner preflight --verify-agent-runtime` and one `/new` + command; follow-ups to pre-cutover sessions start new sessions. +5. Archive or delete the old root separately once the new root is proven. ## 8. Trigger the agent from a comment diff --git a/docs/self-hosting/runner.zh-CN.md b/docs/self-hosting/runner.zh-CN.md index 072f5316..bb8f1195 100644 --- a/docs/self-hosting/runner.zh-CN.md +++ b/docs/self-hosting/runner.zh-CN.md @@ -431,11 +431,20 @@ NO_PROXY=127.0.0.1,localhost,issues.example.test,code.example.test ### 存储生命周期与单一 owner -Runner 的存储以 `--workspace-root` 为锚点：私有的 `.storage` 目录保存 session runtime 根与 -PROCESS workspace pool 的 sidecar 清单、owner 锁以及首次迁移备份。一个规范化 root 只有一个 -具备破坏性的 owner，因此同一 state/workspace 对只能运行一个 systemd unit。在同一 root 上启动 -第二个 `runner serve`（或 `runner poll`）会在启动时失败并提示「stop the old runner before -starting a new one」；更换实例前务必先停止旧 unit。 +Runner 的存储以 `--workspace-root` 为锚点：私有的 `.storage` 目录保存覆盖所有受管物理资源的 +sidecar 清单——runner 作用域共享 runtime home（`.runner-home//`）、按 job 划分的 +scratch 目录（`.job-scratch//`）与 PROCESS workspace pool，以及旧版本二进制创建的 root +上的 per-session runtime 根（`.sessions//`）——外加 owner 锁与首次迁移备份。一个规范化 +root 只有一个具备破坏性的 owner，因此同一 state/workspace 对只能运行一个 systemd unit。在同一 +root 上启动第二个 `runner serve`（或 `runner poll`）会在启动时失败并提示「stop the old runner +before starting a new one」；更换实例前务必先停止旧 unit。 + +每个被分发的 job 共享其 runtime 作用域（hostname、profile realm、仓库与 runner 身份）的持久 +runtime HOME：`.runner-home//`——agent 的 `home/`、`gh/`、`xdg/`、`codex/` 与 +`acpx-runtime/` 目录，外加一个对外来作用域 fail-closed 的 `scope.json` 绑定——因此包与构建缓存 +能让同作用域的后续 job 保持温热。每个 job 还会获得一次性 `.job-scratch//`（`TMPDIR`、 +`GOTMPDIR`、`XDG_DATA_HOME`、`XDG_STATE_HOME`），在 job 进入终态时被移除。home 中的 agent +identity 与配置始终受保护；只有可重建的 cache（`~/.cache`、`~/.npm`、`~/go/pkg/mod`）才可被驱逐。 `runner serve` 上有两个可选的存储参数： @@ -454,10 +463,24 @@ sudo -u issue-spec-runner issue-spec --profile team runner storage reconcile \ ``` `--dry-run` 只报告分类与 `would_delete` 动作；`--apply` 在对照重新加载的 runner 状态重新校验 -每一项之后删除符合条件的资源。两种模式都会获取 owner 锁，因此请在停止 unit 后运行该命令。如果 -命令报告 sidecar 处于 report-only 状态（root 身份不同或 schema 更新），它只读清点并以非零码 -退出；除非同时接受丢失所有权与孤儿观察证明，否则不要手工删除 `.storage`。已退役的 PROCESS -pool 仅在被证明为空时才会删除；不确定的 pool 会被保留并给出处置诊断，绝不会被强制放弃。 +每一项之后删除符合条件的资源。当存在共享 runtime home 时，报告会增加 `runtime:` 小节，给出每个 +home 的受保护/cache/未知字节数以及 job scratch 总字节数；`--apply --evict-caches` 还会 reconcile +过期 job scratch 并驱逐每个已记录 home 中可重建的 cache 目录，打印回收的字节数。两种模式都会获取 +owner 锁，因此请在停止 unit 后运行该命令。如果命令报告 sidecar 处于 report-only 状态（root 身份 +不同或 schema 更新），它只读清点并以非零码退出；除非同时接受丢失所有权与孤儿观察证明，否则不要 +手工删除 `.storage`。已退役的 PROCESS pool 仅在被证明为空时才会删除；不确定的 pool 会被保留并 +给出处置诊断，绝不会被强制放弃。 + +把一个 unit 升级到共享 runtime 布局是一次破坏性切换，不是原地升级：新二进制绝不会读取导入、修改 +或删除旧 root；切换前的 session 不可恢复，因为它们的 acpx 记录保存在旧 root 的 per-session +runtime 中。请使用全新的 runner root 完成切换： + +1. 停止旧 unit（`systemctl stop issue-spec-runner`），让在途 job 跑完或先取消它们。 +2. 如需审计，可先行归档旧的 state/workspace root。 +3. 把 unit 指向全新的空 `--state`/`--workspace-root` 对（或先把旧 root 移走），再启动新二进制。 +4. 用 `runner preflight --verify-agent-runtime` 和一条 `/new` 命令验证；切换前 session 的后续 + 对话请开新 session。 +5. 新 root 验证通过后，再单独归档或删除旧 root。 ## 8. 通过评论触发 Agent From 77783bd106f258cd78ed21202d140fb95db81f26 Mon Sep 17 00:00:00 2001 From: johnlanni Date: Sat, 8 Aug 2026 22:58:48 +0800 Subject: [PATCH 11/17] test(storage): acceptance for shared-home Go cache reuse and fresh-root cutover Add the real-toolchain acceptance (short-skipped, requires go on PATH): two jobs sharing one runtime HOME with distinct per-job scratch reuse the HOME-anchored GOCACHE/GOMODCACHE, an offline second build proves the shared module cache is authoritative, and a concurrent build pair with distinct GOTMPDIRs succeeds with identical outputs. The fast cutover acceptance prepares, reconciles, and evicts against a fresh root while a pre-existing old-style .sessions//home fixture in another root is proven byte-identical afterwards. Signed-off-by: johnlanni --- .../storage/runtime_home_acceptance_test.go | 375 ++++++++++++++++++ 1 file changed, 375 insertions(+) create mode 100644 internal/commentrunner/storage/runtime_home_acceptance_test.go diff --git a/internal/commentrunner/storage/runtime_home_acceptance_test.go b/internal/commentrunner/storage/runtime_home_acceptance_test.go new file mode 100644 index 00000000..579139d7 --- /dev/null +++ b/internal/commentrunner/storage/runtime_home_acceptance_test.go @@ -0,0 +1,375 @@ +package storage + +import ( + "archive/zip" + "bytes" + "context" + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + "sync" + "testing" + + "github.com/higress-group/issue-spec/internal/commentrunner/state" +) + +// TestSharedRuntimeHomeReusesGoCachesAcrossJobs is the real-toolchain +// acceptance proof for the runner-scoped shared layout: two jobs sharing one +// runtime HOME but receiving distinct disposable scratch directories reuse the +// HOME-anchored Go build and module caches, including under concurrency. +// Skipped in -short mode and when no go toolchain is on PATH. +func TestSharedRuntimeHomeReusesGoCachesAcrossJobs(t *testing.T) { + if testing.Short() { + t.Skip("acceptance test drives a real go toolchain") + } + if _, err := exec.LookPath("go"); err != nil { + t.Skipf("go toolchain not on PATH: %v", err) + } + + root := t.TempDir() + scope := RuntimeScope{Hostname: "github.com", Repo: "o/r", Runner: "acceptance"} + home, err := PrepareRuntimeHome(root, scope) + if err != nil { + t.Fatalf("prepare runtime home: %v", err) + } + scratchA, err := PrepareJobScratch(root, "job-aaaaaaaaaaaaaaaa") + if err != nil { + t.Fatalf("prepare job A scratch: %v", err) + } + scratchB, err := PrepareJobScratch(root, "job-bbbbbbbbbbbbbbbb") + if err != nil { + t.Fatalf("prepare job B scratch: %v", err) + } + + // A tiny module with exactly one external dependency, served by a local + // file proxy so the acceptance never touches the network. + proxyDir := t.TempDir() + writeFileProxyModule(t, proxyDir, "example.com/dep", "v1.0.0") + moduleDir := t.TempDir() + writeTestFile(t, filepath.Join(moduleDir, "go.mod"), "module example.com/acceptance\n\ngo 1.21\n\nrequire example.com/dep v1.0.0\n") + writeTestFile(t, filepath.Join(moduleDir, "main.go"), `package main + +import ( + "fmt" + + "example.com/dep" +) + +func main() { fmt.Println(dep.Greeting()) } +`) + + runGo := func(scratch JobScratchPaths, dir string, args ...string) string { + t.Helper() + cmd := exec.Command("go", args...) + cmd.Dir = dir + cmd.Env = []string{ + "PATH=" + os.Getenv("PATH"), + "HOME=" + home.Home, + "TMPDIR=" + scratch.Tmp, + "GOTMPDIR=" + scratch.GoTmp, + "XDG_DATA_HOME=" + scratch.XDGData, + "XDG_STATE_HOME=" + scratch.XDGState, + "GOPROXY=file://" + proxyDir, + "GOSUMDB=off", + "GOFLAGS=-mod=mod", + "CGO_ENABLED=0", + } + out, err := cmd.CombinedOutput() + if err != nil { + t.Fatalf("go %s failed: %v\n%s", strings.Join(args, " "), err, out) + } + return string(out) + } + + // Job A warms the shared caches. + runGo(scratchA, moduleDir, "mod", "download") + binA := filepath.Join(t.TempDir(), "bin-a") + runGo(scratchA, moduleDir, "build", "-buildvcs=false", "-o", binA, ".") + + gocache := strings.TrimSpace(runGo(scratchA, moduleDir, "env", "GOCACHE")) + gomodcache := strings.TrimSpace(runGo(scratchA, moduleDir, "env", "GOMODCACHE")) + if gocache == "" || gomodcache == "" { + t.Fatalf("go env caches must be set: GOCACHE=%q GOMODCACHE=%q", gocache, gomodcache) + } + for name, dir := range map[string]string{"GOCACHE": gocache, "GOMODCACHE": gomodcache} { + if rel, err := filepath.Rel(home.Home, dir); err != nil || strings.HasPrefix(rel, "..") { + t.Fatalf("%s=%q must anchor below the shared home %q", name, dir, home.Home) + } + } + if count := countRegularFiles(t, gocache); count == 0 { + t.Fatalf("shared GOCACHE %q is empty after job A build", gocache) + } + if _, err := os.Lstat(filepath.Join(gomodcache, "example.com", "dep@v1.0.0")); err != nil { + t.Fatalf("shared GOMODCACHE %q missing the downloaded module: %v", gomodcache, err) + } + // Job A scratch held only disposable temp content; the caches live in the + // shared home, not the scratch. + if count := countRegularFiles(t, scratchA.GoTmp); count != 0 { + t.Fatalf("job GOTMPDIR must not retain build content after the build: %d files", count) + } + + // Job B: same shared HOME, distinct scratch. The caches resolve to the + // identical directories and the build succeeds without network access — + // the module is already in the shared module cache (GOPROXY=off proves it). + runGoOff := func(scratch JobScratchPaths, args ...string) string { + t.Helper() + cmd := exec.Command("go", args...) + cmd.Dir = moduleDir + cmd.Env = []string{ + "PATH=" + os.Getenv("PATH"), + "HOME=" + home.Home, + "TMPDIR=" + scratch.Tmp, + "GOTMPDIR=" + scratch.GoTmp, + "XDG_DATA_HOME=" + scratch.XDGData, + "XDG_STATE_HOME=" + scratch.XDGState, + "GOPROXY=off", + "GOFLAGS=-mod=mod", + "CGO_ENABLED=0", + } + out, err := cmd.CombinedOutput() + if err != nil { + t.Fatalf("go %s (offline) failed: %v\n%s", strings.Join(args, " "), err, out) + } + return string(out) + } + binB := filepath.Join(t.TempDir(), "bin-b") + runGoOff(scratchB, "build", "-buildvcs=false", "-o", binB, ".") + if got := strings.TrimSpace(runGoOff(scratchB, "env", "GOCACHE")); got != gocache { + t.Fatalf("job B GOCACHE = %q, want the shared %q", got, gocache) + } + if got := strings.TrimSpace(runGoOff(scratchB, "env", "GOMODCACHE")); got != gomodcache { + t.Fatalf("job B GOMODCACHE = %q, want the shared %q", got, gomodcache) + } + dataA, err := os.ReadFile(binA) + if err != nil { + t.Fatal(err) + } + dataB, err := os.ReadFile(binB) + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(dataA, dataB) { + t.Fatalf("jobs sharing one runtime home must reproduce identical build outputs") + } + + // Concurrent pair: two goroutines, one shared HOME, distinct GOTMPDIR. + scratchC, err := PrepareJobScratch(root, "job-cccccccccccccccc") + if err != nil { + t.Fatal(err) + } + scratchD, err := PrepareJobScratch(root, "job-dddddddddddddddd") + if err != nil { + t.Fatal(err) + } + bins := []string{filepath.Join(t.TempDir(), "bin-c"), filepath.Join(t.TempDir(), "bin-d")} + scratches := []JobScratchPaths{scratchC, scratchD} + var wg sync.WaitGroup + errs := make([]error, 2) + for i := range bins { + wg.Add(1) + go func(i int) { + defer wg.Done() + cmd := exec.Command("go", "build", "-buildvcs=false", "-o", bins[i], ".") + cmd.Dir = moduleDir + cmd.Env = []string{ + "PATH=" + os.Getenv("PATH"), + "HOME=" + home.Home, + "TMPDIR=" + scratches[i].Tmp, + "GOTMPDIR=" + scratches[i].GoTmp, + "XDG_DATA_HOME=" + scratches[i].XDGData, + "XDG_STATE_HOME=" + scratches[i].XDGState, + "GOPROXY=off", + "GOFLAGS=-mod=mod", + "CGO_ENABLED=0", + } + if out, err := cmd.CombinedOutput(); err != nil { + errs[i] = fmt.Errorf("concurrent build: %w\n%s", err, out) + } + }(i) + } + wg.Wait() + for _, err := range errs { + if err != nil { + t.Fatal(err) + } + } + dataC, err := os.ReadFile(bins[0]) + if err != nil { + t.Fatal(err) + } + dataD, err := os.ReadFile(bins[1]) + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(dataC, dataD) || !bytes.Equal(dataA, dataC) { + t.Fatalf("concurrent builds on one shared home must produce identical outputs") + } +} + +// writeFileProxyModule publishes a one-file module into a local GOPROXY +// directory layout so tests can exercise the module cache offline. +func writeFileProxyModule(t *testing.T, proxyDir, module, version string) { + t.Helper() + gomod := "module " + module + "\n\ngo 1.21\n" + source := "package dep\n\n// Greeting identifies the dep build.\nfunc Greeting() string { return \"dep-ok\" }\n" + atVersion := module + "@" + version + var buf bytes.Buffer + zw := zip.NewWriter(&buf) + for name, content := range map[string]string{ + atVersion + "/go.mod": gomod, + atVersion + "/dep.go": source, + } { + w, err := zw.Create(name) + if err != nil { + t.Fatal(err) + } + if _, err := w.Write([]byte(content)); err != nil { + t.Fatal(err) + } + } + if err := zw.Close(); err != nil { + t.Fatal(err) + } + base := filepath.Join(proxyDir, module, "@v") + writeTestFile(t, filepath.Join(base, version+".info"), `{"Version":"`+version+`","Time":"2024-01-01T00:00:00Z"}`) + writeTestFile(t, filepath.Join(base, version+".mod"), gomod) + if err := os.MkdirAll(base, 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(base, version+".zip"), buf.Bytes(), 0o600); err != nil { + t.Fatal(err) + } +} + +func writeTestFile(t *testing.T, path, content string) { + t.Helper() + if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path, []byte(content), 0o600); err != nil { + t.Fatal(err) + } +} + +func countRegularFiles(t *testing.T, root string) int { + t.Helper() + count := 0 + if err := filepath.WalkDir(root, func(_ string, entry os.DirEntry, err error) error { + if err != nil { + return err + } + if entry.Type().IsRegular() { + count++ + } + return nil + }); err != nil { + t.Fatal(err) + } + return count +} + +// TestPrepareRuntimeHomeAndScratchNeverTouchForeignRoots is the fast unit-level +// cutover acceptance: prepare, reconcile, and evict against a fresh root while +// a pre-existing old-style root (with a `.sessions//home` fixture) sits +// elsewhere on disk, and prove every byte of the old root is untouched. +func TestPrepareRuntimeHomeAndScratchNeverTouchForeignRoots(t *testing.T) { + newRoot := t.TempDir() + oldRoot := t.TempDir() + oldFixture := filepath.Join(oldRoot, ".sessions", "0123456789abcdef0123456789abcdef", "home") + writeTestFile(t, filepath.Join(oldFixture, ".claude.json"), `{"old":"session"}`) + writeTestFile(t, filepath.Join(oldRoot, ".storage", "marker"), "old sidecar") + before := snapshotTree(t, oldRoot) + + scope := RuntimeScope{Hostname: "github.com", Repo: "o/r", Runner: "cutover"} + home, err := PrepareRuntimeHome(newRoot, scope) + if err != nil { + t.Fatalf("prepare runtime home: %v", err) + } + if rel, relErr := filepath.Rel(newRoot, home.Root); relErr != nil || strings.HasPrefix(rel, "..") { + t.Fatalf("runtime home %q must anchor below the new root %q", home.Root, newRoot) + } + scratch, err := PrepareJobScratch(newRoot, "job-eeeeeeeeeeeeeeee") + if err != nil { + t.Fatalf("prepare job scratch: %v", err) + } + writeTestFile(t, filepath.Join(scratch.Tmp, "leftover"), "stale") + + svc, err := NewService(ServiceConfig{ + WorkspaceRoot: newRoot, + StateLoader: func(context.Context) (state.RunnerState, error) { return state.NewState(), nil }, + }) + if err != nil { + t.Fatalf("NewService: %v", err) + } + defer svc.Close() + if err := svc.RecordRuntimeHome(context.Background(), scope, home); err != nil { + t.Fatalf("record runtime home: %v", err) + } + if err := svc.RecordJobScratch(context.Background(), "o/r", "job-eeeeeeeeeeeeeeee", scratch.Root); err != nil { + t.Fatalf("record job scratch: %v", err) + } + if _, err := svc.ReconcileStorage(context.Background(), true, true); err != nil { + t.Fatalf("reconcile: %v", err) + } + if _, err := svc.ReconcileJobScratch(context.Background(), true); err != nil { + t.Fatalf("reconcile job scratch: %v", err) + } + if _, err := svc.EvictRuntimeCaches(context.Background(), true); err != nil { + t.Fatalf("evict runtime caches: %v", err) + } + // The passes did real work on the new root: the stale scratch of the + // unknown job is reclaimed and the shared home is kept. + if _, err := os.Lstat(scratch.Root); !os.IsNotExist(err) { + t.Fatalf("stale scratch must be reclaimed on the new root, err=%v", err) + } + if _, err := os.Lstat(filepath.Join(home.Root, "scope.json")); err != nil { + t.Fatalf("shared runtime home must survive reconciliation: %v", err) + } + + after := snapshotTree(t, oldRoot) + if !equalStringMaps(before, after) { + t.Fatalf("old-style root was read-for-import, modified, or deleted:\nbefore=%v\nafter=%v", before, after) + } +} + +// snapshotTree maps every file below root to its content so a byte-exact +// before/after comparison proves the tree was never touched. +func snapshotTree(t *testing.T, root string) map[string]string { + t.Helper() + out := map[string]string{} + if err := filepath.WalkDir(root, func(path string, entry os.DirEntry, err error) error { + if err != nil { + return err + } + if entry.IsDir() || !entry.Type().IsRegular() { + return nil + } + data, readErr := os.ReadFile(path) + if readErr != nil { + return readErr + } + rel, relErr := filepath.Rel(root, path) + if relErr != nil { + return relErr + } + out[rel] = string(data) + return nil + }); err != nil { + t.Fatal(err) + } + return out +} + +func equalStringMaps(a, b map[string]string) bool { + if len(a) != len(b) { + return false + } + for k, v := range a { + if other, ok := b[k]; !ok || other != v { + return false + } + } + return true +} From 5528b5a73c3058c924395fac8110887f0bc7c854 Mon Sep 17 00:00:00 2001 From: johnlanni Date: Sat, 8 Aug 2026 22:59:55 +0800 Subject: [PATCH 12/17] test(runner): prove dispatch on a fresh root never touches old-style roots Extend the shared-layout suite with the dispatch-side cutover acceptance: a shared-layout dispatch against a fresh runner root selects the runner-scoped shared HOME while a pre-existing .sessions//home fixture in a separate old-style root stays byte-identical and gains no entries. Signed-off-by: johnlanni --- .../jobs/shared_layout_dispatch_test.go | 53 +++++++++++++++++++ 1 file changed, 53 insertions(+) diff --git a/internal/commentrunner/jobs/shared_layout_dispatch_test.go b/internal/commentrunner/jobs/shared_layout_dispatch_test.go index 5f3529d5..cef1c5ff 100644 --- a/internal/commentrunner/jobs/shared_layout_dispatch_test.go +++ b/internal/commentrunner/jobs/shared_layout_dispatch_test.go @@ -1,6 +1,7 @@ package jobs import ( + "bytes" "context" "errors" "fmt" @@ -238,6 +239,58 @@ func TestSharedLayoutRuntimeHomeDiffersAcrossRepos(t *testing.T) { } } +// TestSharedLayoutDispatchNeverTouchesOldStyleRoots is the dispatch-side +// fresh-root cutover acceptance: dispatching on a fresh root selects the +// shared HOME, while a pre-existing old-style `.sessions//home` fixture +// in a separate root is never read for import, modified, or deleted. +func TestSharedLayoutDispatchNeverTouchesOldStyleRoots(t *testing.T) { + store := newMemoryStore() + now := time.Date(2026, 7, 6, 9, 45, 0, 0, time.UTC) + root, err := filepath.EvalSymlinks(t.TempDir()) + if err != nil { + t.Fatal(err) + } + oldRoot := t.TempDir() + oldFixture := filepath.Join(oldRoot, ".sessions", "0123456789abcdef0123456789abcdef", "home") + if err := os.MkdirAll(oldFixture, 0o700); err != nil { + t.Fatal(err) + } + oldBytes := []byte("{\"old\":\"session runtime\"}\n") + if err := os.WriteFile(filepath.Join(oldFixture, ".claude.json"), oldBytes, 0o600); err != nil { + t.Fatal(err) + } + + workspaces := &fakeWorkspaces{binding: sharedLayoutBinding(t, root, "ws-cutover", "o/r")} + coordinator := &fakeCoordinator{newResult: dispatchResult("ps-cutover", "rec-cutover", "turn-cutover", completedSummary())} + dispatcher, sandbox := newSharedLayoutDispatcher(store, workspaces, coordinator, &fakeStorage{}, now) + + seedSharedLayoutJob(t, store, "job-9999999999999999", "o/r", now) + result, err := dispatcher.RunNext(context.Background()) + if err != nil || !result.Executed || result.Status != state.StatusCompleted { + t.Fatalf("cutover dispatch result=%+v err=%v", result, err) + } + if len(sandbox.requests) != 1 { + t.Fatalf("sandbox requests = %d, want 1", len(sandbox.requests)) + } + homeRoot, err := storage.RuntimeHomeRoot(root, sharedLayoutIdentity().scope("o/r")) + if err != nil { + t.Fatal(err) + } + if sandbox.requests[0].RuntimeHome != filepath.Join(homeRoot, "home") { + t.Fatalf("fresh-root dispatch must select the shared HOME %q, got %q", + filepath.Join(homeRoot, "home"), sandbox.requests[0].RuntimeHome) + } + + got, err := os.ReadFile(filepath.Join(oldFixture, ".claude.json")) + if err != nil || !bytes.Equal(got, oldBytes) { + t.Fatalf("old-style root fixture changed: bytes=%q err=%v", got, err) + } + entries, err := os.ReadDir(oldRoot) + if err != nil || len(entries) != 1 || entries[0].Name() != ".sessions" { + t.Fatalf("old-style root gained or lost entries: entries=%v err=%v", entries, err) + } +} + // TestZeroRuntimeIdentityKeepsLegacySessionLayout pins the zero-identity // behavior: existing wiring without a runner runtime identity keeps the // per-session .sessions/ runtime and receives no job scratch. From d7edc31250e99157c7366518ca0066340735f262 Mon Sep 17 00:00:00 2001 From: johnlanni Date: Sat, 8 Aug 2026 23:56:29 +0800 Subject: [PATCH 13/17] fix(runner): harden shared-home scratch recovery and cache eviction Independent review follow-ups for the runner-scoped shared runtime home: - The dispatcher's periodic storage reconcile now also reclaims crash-leftover job scratch on the shared layout, best-effort with a bounded diagnostic, instead of only under disk pressure or through the operator CLI. - ReconcileJobScratch revalidates each job against freshly loaded runner state immediately before deletion (the engine's D8 discipline): a job that turned active or newly known since the pass snapshot aborts its deletion, and a reload failure fails safe. - EvictRuntimeCaches skips homes whose scope has any active job so pressured eviction never breaks in-flight builds, and reports a bounded deferral diagnostic when every home is in use. - runner storage reconcile --json always emits the wrapped schema (report plus optional runtime/eviction sections). - Comment notes: stranded .issue-spec-mirror-* temps are bounded and self-healing, the child-profile gh-dir wipe is a steady-state no-op, and RuntimeIdentityFor inherits profile-resolution stability requirements. - Tests cover the periodic scratch recovery, deletion-time revalidation races, the eviction in-use guard, the stable JSON schema, and intermediate-symlink eviction redirection. Signed-off-by: johnlanni --- internal/commands/runner_storage.go | 28 ++- internal/commands/runner_storage_test.go | 23 +- internal/commentrunner/jobs/dispatcher.go | 22 +- internal/commentrunner/jobs/reconcile.go | 14 +- .../jobs/storage_integration_test.go | 144 ++++++++++-- .../storage/runtime_reconcile.go | 119 ++++++++-- .../storage/runtime_reconcile_test.go | 221 +++++++++++++++++- 7 files changed, 516 insertions(+), 55 deletions(-) diff --git a/internal/commands/runner_storage.go b/internal/commands/runner_storage.go index 8576e872..3e21a6df 100644 --- a/internal/commands/runner_storage.go +++ b/internal/commands/runner_storage.go @@ -79,6 +79,9 @@ func (f failingStorageLifecycle) CompleteJobScratch(context.Context, string, str func (f failingStorageLifecycle) ReconcileStorage(context.Context, bool, bool) (storage.Report, error) { return storage.Report{}, f.err } +func (f failingStorageLifecycle) ReconcileJobScratch(context.Context, bool) (storage.RuntimeReconcileReport, error) { + return storage.RuntimeReconcileReport{}, f.err +} func (a *app) runRunnerStorage(ctx context.Context, args []string) int { if len(args) == 0 { @@ -120,7 +123,10 @@ Options: --apply apply recoverable deletions --evict-caches with --apply: also reconcile stale job scratch and evict rebuildable runtime caches, printing reclaimed bytes - --json write the JSON report`) + --json write the JSON report; the schema is stable: the + reconcile report is always nested under "report", + with "runtime" and "eviction" sections present only + when runtime data or --evict-caches results exist`) } func (a *app) runRunnerStorageReconcile(ctx context.Context, args []string) int { @@ -135,7 +141,7 @@ func (a *app) runRunnerStorageReconcile(ctx context.Context, args []string) int dryRun := fs.Bool("dry-run", false, "report only, no mutations") apply := fs.Bool("apply", false, "apply recoverable deletions") evictCaches := fs.Bool("evict-caches", false, "with --apply: reconcile stale job scratch and evict rebuildable runtime caches") - jsonOut := fs.Bool("json", false, "write JSON output") + jsonOut := fs.Bool("json", false, "write JSON output (stable schema: report nested under \"report\", optional runtime/eviction sections)") fs.Var(&repoValues, "repo", "repository owner/name for scoped defaults; repeat or comma-separate") if argsContainHelp(args) { fs.SetOutput(a.out) @@ -273,10 +279,7 @@ func (a *app) runRunnerStorageReconcile(ctx context.Context, args []string) int } runtimeUsage := collectRuntimeUsage(service) if *jsonOut { - if runtimeUsage != nil || eviction != nil { - return a.outputJSON(runnerStorageReconcileOutput{Report: report, Runtime: runtimeUsage, Eviction: eviction}) - } - return a.outputJSON(report) + return a.outputJSON(runnerStorageReconcileOutput{Report: report, Runtime: runtimeUsage, Eviction: eviction}) } printStorageReport(a.out, report, runtimeUsage, eviction) if report.ReportOnly { @@ -285,8 +288,11 @@ func (a *app) runRunnerStorageReconcile(ctx context.Context, args []string) int return 0 } -// runnerStorageReconcileOutput is the JSON shape when the runtime usage -// section or the --evict-caches passes accompany the main reconcile report. +// runnerStorageReconcileOutput is the stable --json schema of runner storage +// reconcile: the main report is always nested under "report", and the +// "runtime"/"eviction" sections appear only when runtime home measurements or +// an --evict-caches pass exist. Consumers always decode the same top-level +// shape regardless of which sections a given run produced. type runnerStorageReconcileOutput struct { Report storage.Report `json:"report"` Runtime *runtimeUsageSection `json:"runtime,omitempty"` @@ -319,9 +325,9 @@ type runtimeEvictionReport struct { // collectRuntimeUsage measures every recorded runner home and the job scratch // base. It returns nil when no runner home records exist, so legacy-layout -// roots keep the previous report shape. Measurement failures degrade to -// diagnostics: the reconcile pass already succeeded and its report stays -// valid. +// roots keep their previous human output and simply omit the JSON runtime +// section. Measurement failures degrade to diagnostics: the reconcile pass +// already succeeded and its report stays valid. func collectRuntimeUsage(service *storage.Service) *runtimeUsageSection { records := make([]storage.PhysicalResource, 0) for _, resource := range service.Store().State().Resources { diff --git a/internal/commands/runner_storage_test.go b/internal/commands/runner_storage_test.go index ef598101..2d92975d 100644 --- a/internal/commands/runner_storage_test.go +++ b/internal/commands/runner_storage_test.go @@ -118,10 +118,27 @@ func TestRunnerStorageReconcileJSON(t *testing.T) { if code != 0 { t.Fatalf("exit=%d stderr=%q", code, errOut.String()) } - var report storage.Report - if err := json.Unmarshal(out.Bytes(), &report); err != nil { - t.Fatalf("decode report: %v\n%s", err, out.String()) + // The JSON schema is stable: the report is always wrapped, even on a + // legacy root with no runtime homes or eviction results. + var top map[string]json.RawMessage + if err := json.Unmarshal(out.Bytes(), &top); err != nil { + t.Fatalf("decode top-level report: %v\n%s", err, out.String()) + } + if _, ok := top["report"]; !ok { + t.Fatalf("JSON must always wrap the report under \"report\": %s", out.String()) + } + for _, absent := range []string{"runtime", "eviction"} { + if _, ok := top[absent]; ok { + t.Fatalf("legacy root must omit the %q section: %s", absent, out.String()) + } + } + var wrapped struct { + Report storage.Report `json:"report"` + } + if err := json.Unmarshal(out.Bytes(), &wrapped); err != nil { + t.Fatalf("decode wrapped report: %v\n%s", err, out.String()) } + report := wrapped.Report if !report.DryRun { t.Fatalf("report not marked dry-run") } diff --git a/internal/commentrunner/jobs/dispatcher.go b/internal/commentrunner/jobs/dispatcher.go index 99673b30..bcf06e4b 100644 --- a/internal/commentrunner/jobs/dispatcher.go +++ b/internal/commentrunner/jobs/dispatcher.go @@ -146,6 +146,9 @@ type StorageLifecycle interface { RecordJobScratch(ctx context.Context, repo, jobID, path string) error CompleteJobScratch(ctx context.Context, repo, jobID string) error ReconcileStorage(ctx context.Context, apply, measureAll bool) (storage.Report, error) + // ReconcileJobScratch reclaims crash-leftover disposable scratch of + // terminal or unknown jobs while keeping active-job scratch. + ReconcileJobScratch(ctx context.Context, apply bool) (storage.RuntimeReconcileReport, error) } // RuntimeIdentity pins the dispatcher to one runner scope: every job of a @@ -175,7 +178,10 @@ func (r RuntimeIdentity) scope(repo string) storage.RuntimeScope { // RuntimeIdentityFor derives the dispatcher runtime identity from the runner // configuration: the named profile is resolved exactly as the runner scope // path derivation resolves it, then RuntimeIdentityFromProfile computes the -// identity. +// identity. It inherits the profile-resolution stability requirements of the +// runner scope paths: restart the runner from a consistent working directory +// (or pin an explicit profile) so the same scope — and therefore the same +// shared runtime home — is resolved every time. func RuntimeIdentityFor(hostname, profileName, runner string) (RuntimeIdentity, error) { profile, _, err := clientauth.ResolveProfile(profileName, hostname) if err != nil { @@ -1928,8 +1934,11 @@ func (p SandboxRunner) config(req SandboxRequest) (sandbox.Config, string, ghAut } // A self-hosted child has no reason to observe hosts.yml or shared gh // state, even if the operator process uses it for legacy GitHub mode. - // Content-aware: on the shared runtime home this dir is already empty - // in steady state, and wiping it anyway would race concurrent jobs. + // Invariant: in child-profile mode mirrorHostGHAuth never runs, so the + // shared gh dir is always empty and the wipe below is a steady-state + // no-op. Content-aware: on the shared runtime home this dir is already + // empty in steady state, and wiping it anyway would race concurrent + // jobs. empty, err := dirEmpty(cfg.TempGHConfigDir) if err != nil { return sandbox.Config{}, "", ghAuthMirror, err @@ -2065,7 +2074,12 @@ func dirEmpty(dir string) (bool, error) { // directory plus rename, so a concurrent reader of the shared runtime home // never observes a partial mirror. An existing identical regular file is left // untouched (mode enforced): steady-state refreshes must not churn the shared -// home. +// home. A crash between temp creation and rename may strand a bounded +// `.issue-spec-mirror-*` temp file in the destination directory — at most one +// per crashed refresh, mirror-payload sized, dot-prefixed so it is never a +// read target. The mirror state is self-healing: the next refresh of the same +// path completes the interrupted install with a fresh atomic rename, so no +// operator cleanup is ever required. func writeFileAtomic(path string, data []byte, mode os.FileMode) error { if info, err := os.Lstat(path); err == nil && info.Mode().IsRegular() { if existing, readErr := os.ReadFile(path); readErr == nil && bytes.Equal(existing, data) { diff --git a/internal/commentrunner/jobs/reconcile.go b/internal/commentrunner/jobs/reconcile.go index 382f4d39..f0c75374 100644 --- a/internal/commentrunner/jobs/reconcile.go +++ b/internal/commentrunner/jobs/reconcile.go @@ -152,7 +152,11 @@ func (d *Dispatcher) CleanupWorkspaces(ctx context.Context) (ReconcileResult, er // reconcileStoragePass runs the shared storage reconciliation before workspace // cleanup so runtime deletion precedes grouped workspace deletion. Runtime // deletion failures defer their grouped workspace IDs this pass. A storage -// failure is diagnosed, never fatal to control-plane reconciliation. +// failure is diagnosed, never fatal to control-plane reconciliation. On the +// shared runtime layout the pass also recovers crash-leftover job scratch on +// every cycle — not only under disk pressure or through the operator CLI — +// best-effort: a scratch failure is a bounded diagnostic, never a failed +// pass. func (d *Dispatcher) reconcileStoragePass(ctx context.Context) (*storage.Report, map[string]bool) { if d.Storage == nil { return nil, nil @@ -161,6 +165,14 @@ func (d *Dispatcher) reconcileStoragePass(ctx context.Context) (*storage.Report, if err != nil { return &storage.Report{Diagnostics: []string{"storage reconciliation: " + safeError(err)}}, nil } + if d.RuntimeIdentity.enabled() { + scratch, scratchErr := d.Storage.ReconcileJobScratch(ctx, true) + if scratchErr != nil { + report.Diagnostics = append(report.Diagnostics, "job scratch reconciliation: "+safeError(scratchErr)) + } else { + report.Diagnostics = append(report.Diagnostics, scratch.Diagnostics...) + } + } var deferred map[string]bool for _, id := range report.DeferredWorkspaceIDs { if id == "" { diff --git a/internal/commentrunner/jobs/storage_integration_test.go b/internal/commentrunner/jobs/storage_integration_test.go index ed5fa608..99c9b0db 100644 --- a/internal/commentrunner/jobs/storage_integration_test.go +++ b/internal/commentrunner/jobs/storage_integration_test.go @@ -37,22 +37,26 @@ type recordedScratchCompletion struct { } type fakeStorage struct { - admitErr error - recordErr error - recordCalls []recordedSessionResource - poolErr error - poolCalls []recordedSessionResource - homeErr error - homeCalls []recordedRuntimeHome - scratchErr error - scratchCalls []recordedJobScratch - completeErr error - completeCalls []recordedScratchCompletion - admitCalls int - reconcileCalls int - reconcileApply []bool - reconcileReport storage.Report - reconcileErr error + admitErr error + recordErr error + recordCalls []recordedSessionResource + poolErr error + poolCalls []recordedSessionResource + homeErr error + homeCalls []recordedRuntimeHome + scratchErr error + scratchCalls []recordedJobScratch + completeErr error + completeCalls []recordedScratchCompletion + admitCalls int + reconcileCalls int + reconcileApply []bool + reconcileReport storage.Report + reconcileErr error + scratchReconcileCalls int + scratchReconcileApply []bool + scratchReconcileReport storage.RuntimeReconcileReport + scratchReconcileErr error } func (f *fakeStorage) AdmitDispatch(context.Context) error { @@ -91,6 +95,12 @@ func (f *fakeStorage) ReconcileStorage(_ context.Context, apply, _ bool) (storag return f.reconcileReport, f.reconcileErr } +func (f *fakeStorage) ReconcileJobScratch(_ context.Context, apply bool) (storage.RuntimeReconcileReport, error) { + f.scratchReconcileCalls++ + f.scratchReconcileApply = append(f.scratchReconcileApply, apply) + return f.scratchReconcileReport, f.scratchReconcileErr +} + func TestRunJobStoragePressureDelaysWithoutFailing(t *testing.T) { store := newMemoryStore() now := time.Date(2026, 7, 3, 13, 0, 0, 0, time.UTC) @@ -239,6 +249,108 @@ func TestReconcileStorageFailureIsNonFatal(t *testing.T) { } } +func TestReconcileInvokesJobScratchReconcileOnSharedLayout(t *testing.T) { + store := newMemoryStore() + now := time.Date(2026, 7, 3, 13, 0, 0, 0, time.UTC) + dispatcher := testDispatcher(store, &fakeWorkspaces{binding: testBinding("ws-1")}, &fakeCoordinator{}, &fakeWriteback{}, now) + fake := &fakeStorage{} + dispatcher.Storage = fake + dispatcher.RuntimeIdentity = sharedLayoutIdentity() + + if _, err := dispatcher.Reconcile(context.Background()); err != nil { + t.Fatalf("Reconcile: %v", err) + } + if fake.scratchReconcileCalls != 1 || len(fake.scratchReconcileApply) != 1 || !fake.scratchReconcileApply[0] { + t.Fatalf("shared-layout reconcile must run one applied scratch pass, calls=%d apply=%v", + fake.scratchReconcileCalls, fake.scratchReconcileApply) + } +} + +func TestReconcileSkipsJobScratchReconcileOnLegacyLayout(t *testing.T) { + store := newMemoryStore() + now := time.Date(2026, 7, 3, 13, 0, 0, 0, time.UTC) + dispatcher := testDispatcher(store, &fakeWorkspaces{binding: testBinding("ws-1")}, &fakeCoordinator{}, &fakeWriteback{}, now) + fake := &fakeStorage{} + dispatcher.Storage = fake + + if _, err := dispatcher.Reconcile(context.Background()); err != nil { + t.Fatalf("Reconcile: %v", err) + } + if fake.scratchReconcileCalls != 0 { + t.Fatalf("legacy layout has no job scratch to recover, calls=%d", fake.scratchReconcileCalls) + } +} + +func TestReconcileJobScratchFailureIsNonFatal(t *testing.T) { + store := newMemoryStore() + now := time.Date(2026, 7, 3, 13, 0, 0, 0, time.UTC) + dispatcher := testDispatcher(store, &fakeWorkspaces{binding: testBinding("ws-1")}, &fakeCoordinator{}, &fakeWriteback{}, now) + dispatcher.Storage = &fakeStorage{scratchReconcileErr: errors.New("scratch sweep failed")} + dispatcher.RuntimeIdentity = sharedLayoutIdentity() + + result, err := dispatcher.Reconcile(context.Background()) + if err != nil { + t.Fatalf("job scratch failure must not fail job reconcile: %v", err) + } + found := false + for _, d := range result.Diagnostics { + if strings.Contains(d, "job scratch reconciliation") { + found = true + } + } + if !found { + t.Fatalf("expected bounded job scratch diagnostic, got %+v", result.Diagnostics) + } +} + +// TestReconcileStoragePassRecoversStaleJobScratch runs the periodic storage +// pass against a real storage service: crash-leftover scratch of a terminal +// job is reclaimed automatically while an active job's scratch survives. +func TestReconcileStoragePassRecoversStaleJobScratch(t *testing.T) { + store := newMemoryStore() + root := t.TempDir() + seedState(t, store, func(st *state.RunnerState) error { + st.Jobs["job-aaaaaaaaaaaaaaaa"] = state.Job{ID: "job-aaaaaaaaaaaaaaaa", Repo: "o/r", Status: state.StatusRunning} + st.Jobs["job-bbbbbbbbbbbbbbbb"] = state.Job{ID: "job-bbbbbbbbbbbbbbbb", Repo: "o/r", Status: state.StatusCompleted} + return nil + }) + svc, err := storage.NewService(storage.ServiceConfig{ + WorkspaceRoot: root, + StateLoader: store.Load, + }) + if err != nil { + t.Fatalf("NewService: %v", err) + } + defer svc.Close() + activeScratch, err := storage.PrepareJobScratch(root, "job-aaaaaaaaaaaaaaaa") + if err != nil { + t.Fatalf("prepare active scratch: %v", err) + } + terminalScratch, err := storage.PrepareJobScratch(root, "job-bbbbbbbbbbbbbbbb") + if err != nil { + t.Fatalf("prepare terminal scratch: %v", err) + } + for jobID, paths := range map[string]storage.JobScratchPaths{ + "job-aaaaaaaaaaaaaaaa": activeScratch, + "job-bbbbbbbbbbbbbbbb": terminalScratch, + } { + if err := svc.RecordJobScratch(context.Background(), "o/r", jobID, paths.Root); err != nil { + t.Fatalf("RecordJobScratch %s: %v", jobID, err) + } + } + dispatcher := &Dispatcher{Storage: svc, RuntimeIdentity: sharedLayoutIdentity()} + report, _ := dispatcher.reconcileStoragePass(context.Background()) + if report == nil { + t.Fatalf("storage report is required") + } + if _, err := os.Lstat(terminalScratch.Root); !os.IsNotExist(err) { + t.Fatalf("periodic pass must reclaim terminal-job scratch, err=%v", err) + } + if _, err := os.Lstat(activeScratch.Root); err != nil { + t.Fatalf("periodic pass must keep active-job scratch: %v", err) + } +} + func TestCleanupWorkspacesInvokesStorageReconcile(t *testing.T) { store := newMemoryStore() now := time.Date(2026, 7, 3, 13, 0, 0, 0, time.UTC) diff --git a/internal/commentrunner/storage/runtime_reconcile.go b/internal/commentrunner/storage/runtime_reconcile.go index 13ecc40b..37a98e38 100644 --- a/internal/commentrunner/storage/runtime_reconcile.go +++ b/internal/commentrunner/storage/runtime_reconcile.go @@ -25,10 +25,11 @@ type RuntimeReconcileReport struct { Diagnostics []string `json:"diagnostics,omitempty"` } -// jobScratchActive mirrors buildProtectionView's active job statuses: an +// jobRuntimeActive mirrors buildProtectionView's active job statuses: an // interrupted job still counts as active because its sandbox may be reaped -// asynchronously, so its scratch is never reclaimed underneath it. -func jobScratchActive(status state.LifecycleStatus) bool { +// asynchronously, so its scratch is never reclaimed underneath it and its +// scope's runtime home caches are never evicted mid-build. +func jobRuntimeActive(status state.LifecycleStatus) bool { switch status { case state.StatusQueued, state.StatusDispatched, state.StatusRunning, state.StatusInterrupted: return true @@ -89,8 +90,11 @@ func validateScratchDeletionTarget(workspaceRoot, target, jobID string) error { // ReconcileJobScratch removes scratch of terminal or unknown jobs and keeps // scratch of active jobs, both for recorded entries and for on-disk leftovers // a crashed runner never recorded. Foreign names below `.job-scratch` are -// rejected and never deleted. The pass is idempotent: a second apply is a -// no-op. +// rejected and never deleted. Every deletion revalidates its job against +// freshly loaded runner state immediately before removal (the engine's D8 +// deletion-time discipline): a job that turned active or newly known since +// the pass snapshot aborts its deletion. The pass is idempotent: a second +// apply is a no-op. func (s *Service) ReconcileJobScratch(ctx context.Context, apply bool) (RuntimeReconcileReport, error) { _, release, err := EnsureOwner(ctx, s.root) if err != nil { @@ -103,8 +107,10 @@ func (s *Service) ReconcileJobScratch(ctx context.Context, apply bool) (RuntimeR } st.Normalize() active := map[string]bool{} + known := map[string]bool{} for _, job := range st.Jobs { - if jobScratchActive(job.Status) { + known[job.ID] = true + if jobRuntimeActive(job.Status) { active[job.ID] = true } } @@ -141,18 +147,7 @@ func (s *Service) ReconcileJobScratch(ctx context.Context, apply bool) (RuntimeR if active[jobID] { report.ScratchKept = append(report.ScratchKept, jobID) if mutate && record.CleanupState != CleanupManaged { - // A crash-interrupted completion raced a still-active job: - // heal the record instead of deleting live scratch. - if err := s.store.Update(func(st *StorageState) error { - current, ok := st.Resources[id] - if ok { - current.CleanupState = CleanupManaged - st.Resources[id] = current - } - return nil - }); err != nil { - report.Diagnostics = append(report.Diagnostics, safeDiagnostic("job scratch "+jobID+" heal failed: "+err.Error())) - } + s.healScratchRecord(id, jobID, &report) } continue } @@ -176,6 +171,20 @@ func (s *Service) ReconcileJobScratch(ctx context.Context, apply bool) (RuntimeR report.Diagnostics = append(report.Diagnostics, "would remove job scratch "+jobID) continue } + abort, revalidateErr := s.revalidateScratchDeletion(ctx, jobID, known[jobID]) + if revalidateErr != nil { + report.Diagnostics = append(report.Diagnostics, safeDiagnostic("job scratch "+jobID+" deletion skipped: "+revalidateErr.Error())) + continue + } + if abort { + // The job turned active or newly known after the pass snapshot: + // keep its scratch and heal the record like the active path. + report.ScratchKept = append(report.ScratchKept, jobID) + if record.CleanupState != CleanupManaged { + s.healScratchRecord(id, jobID, &report) + } + continue + } if err := s.removeJobScratch(record, &report); err != nil { report.Diagnostics = append(report.Diagnostics, safeDiagnostic("job scratch "+jobID+" removal failed: "+err.Error())) } @@ -210,6 +219,15 @@ func (s *Service) ReconcileJobScratch(ctx context.Context, apply bool) (RuntimeR report.Diagnostics = append(report.Diagnostics, "would remove unrecorded job scratch "+name) continue } + abort, revalidateErr := s.revalidateScratchDeletion(ctx, name, known[name]) + if revalidateErr != nil { + report.Diagnostics = append(report.Diagnostics, safeDiagnostic("job scratch entry "+name+" deletion skipped: "+revalidateErr.Error())) + continue + } + if abort { + report.ScratchKept = append(report.ScratchKept, name) + continue + } measured := measureTreeBytes(path) if err := removeOpenedTree(path, nil); err != nil { report.Diagnostics = append(report.Diagnostics, safeDiagnostic("job scratch entry "+name+" removal failed: "+err.Error())) @@ -225,6 +243,46 @@ func (s *Service) ReconcileJobScratch(ctx context.Context, apply bool) (RuntimeR return report, nil } +// revalidateScratchDeletion reloads runner state immediately before one +// scratch deletion, mirroring the engine's D8 deletion-time revalidation: a +// job that became active — or became known at all after the pass snapshot +// classified it unknown — aborts its deletion this pass. wasKnown reports +// whether the pass snapshot held the job in state. A reload failure aborts +// the deletion fail-safe: the scratch survives and is retried on a later +// pass. +func (s *Service) revalidateScratchDeletion(ctx context.Context, jobID string, wasKnown bool) (bool, error) { + fresh, err := s.stateLoader(ctx) + if err != nil { + return true, fmt.Errorf("deletion-time state reload: %w", err) + } + fresh.Normalize() + job, knownNow := fresh.Jobs[jobID] + switch { + case knownNow && jobRuntimeActive(job.Status): + return true, nil + case knownNow && !wasKnown: + return true, nil + default: + return false, nil + } +} + +// healScratchRecord returns a live job's scratch record to managed state: a +// crash-interrupted completion raced a still-active job, so the record is +// healed instead of deleting live scratch. +func (s *Service) healScratchRecord(id, jobID string, report *RuntimeReconcileReport) { + if err := s.store.Update(func(st *StorageState) error { + current, ok := st.Resources[id] + if ok { + current.CleanupState = CleanupManaged + st.Resources[id] = current + } + return nil + }); err != nil { + report.Diagnostics = append(report.Diagnostics, safeDiagnostic("job scratch "+jobID+" heal failed: "+err.Error())) + } +} + // removeJobScratch runs the recorded-entry lifecycle: mark deleting, remove // the directory capability scoped, then garbage-collect the record. func (s *Service) removeJobScratch(record PhysicalResource, report *RuntimeReconcileReport) error { @@ -256,13 +314,28 @@ func (s *Service) removeJobScratch(record PhysicalResource, report *RuntimeRecon // EvictRuntimeCaches removes only the rebuildable cache subtrees of every // recorded runtime home, in eviction priority order. Protected identity and -// configuration paths are never deletion targets. +// configuration paths are never deletion targets. A home whose scope has any +// active (queued/dispatched/running/interrupted) job is skipped whole: +// pressured eviction must not break in-flight builds. When every recorded +// home is skipped for that reason the pass reports the deferral so an +// operator (or the pressured-admission path) can tell eviction did run. func (s *Service) EvictRuntimeCaches(ctx context.Context, apply bool) (RuntimeReconcileReport, error) { _, release, err := EnsureOwner(ctx, s.root) if err != nil { return RuntimeReconcileReport{}, fmt.Errorf("runtime cache eviction owner: %w", err) } defer release() + st, err := s.stateLoader(ctx) + if err != nil { + return RuntimeReconcileReport{}, fmt.Errorf("load runner state for runtime cache eviction: %w", err) + } + st.Normalize() + activeRepos := map[string]bool{} + for _, job := range st.Jobs { + if jobRuntimeActive(job.Status) { + activeRepos[strings.TrimSpace(job.Repo)] = true + } + } if err := s.store.Reload(); err != nil { return RuntimeReconcileReport{}, fmt.Errorf("reload storage sidecar: %w", err) } @@ -283,8 +356,13 @@ func (s *Service) EvictRuntimeCaches(ctx context.Context, apply bool) (RuntimeRe ids = append(ids, id) } sort.Strings(ids) + deferredActive := 0 for _, id := range ids { home := homes[id] + if activeRepos[strings.TrimSpace(home.Repo)] { + deferredActive++ + continue + } if err := s.validateRuntimeHomeRecord(home); err != nil { report.Diagnostics = append(report.Diagnostics, safeDiagnostic("runtime home "+home.PhysicalHash+" record fails validation: "+err.Error())) continue @@ -316,6 +394,9 @@ func (s *Service) EvictRuntimeCaches(ctx context.Context, apply bool) (RuntimeRe report.ReclaimedBytes += measured } } + if deferredActive > 0 && deferredActive == len(ids) { + report.Diagnostics = append(report.Diagnostics, safeDiagnostic(fmt.Sprintf("runtime cache eviction deferred: all %d recorded runner home(s) have active jobs; sessions are active", deferredActive))) + } sort.Strings(report.CacheEvicted) return report, nil } diff --git a/internal/commentrunner/storage/runtime_reconcile_test.go b/internal/commentrunner/storage/runtime_reconcile_test.go index a85f3d08..3f98e8e1 100644 --- a/internal/commentrunner/storage/runtime_reconcile_test.go +++ b/internal/commentrunner/storage/runtime_reconcile_test.go @@ -2,6 +2,7 @@ package storage import ( "context" + "errors" "os" "path/filepath" "strings" @@ -18,11 +19,16 @@ const ( ) func newRuntimeService(t *testing.T, st state.RunnerState) (*Service, string) { + t.Helper() + return newRuntimeServiceWithLoader(t, func(context.Context) (state.RunnerState, error) { return st, nil }) +} + +func newRuntimeServiceWithLoader(t *testing.T, loader StateLoader) (*Service, string) { t.Helper() root := testRoot(t) svc, err := NewService(ServiceConfig{ WorkspaceRoot: root, - StateLoader: func(context.Context) (state.RunnerState, error) { return st, nil }, + StateLoader: loader, OrphanGrace: DefaultOrphanGrace, }) if err != nil { @@ -205,6 +211,91 @@ func TestReconcileJobScratchDryRunDoesNotMutate(t *testing.T) { } } +// TestReconcileJobScratchRevalidatesBeforeDeletion simulates jobs turning +// active between the pass snapshot and the deletion, using the engine test's +// flip-loader pattern: the snapshot load classifies one recorded terminal job +// and one unrecorded unknown leftover as deletion-eligible, and every +// deletion-time reload sees them running again, so both scratch trees must +// survive. +func TestReconcileJobScratchRevalidatesBeforeDeletion(t *testing.T) { + before := state.NewState() + before.Jobs[scratchJobDone] = state.Job{ID: scratchJobDone, Repo: "o/r", Status: state.StatusCompleted} + after := state.NewState() + after.Jobs[scratchJobDone] = state.Job{ID: scratchJobDone, Repo: "o/r", Status: state.StatusRunning} + after.Jobs[scratchJobOrphan] = state.Job{ID: scratchJobOrphan, Repo: "o/r", Status: state.StatusDispatched} + svc, root := newRuntimeServiceWithLoader(t, flipLoader(before, after)) + donePaths := prepareScratchWithFile(t, root, scratchJobDone, 100) + orphanPaths := prepareScratchWithFile(t, root, scratchJobOrphan, 50) + if err := svc.RecordJobScratch(context.Background(), "o/r", scratchJobDone, donePaths.Root); err != nil { + t.Fatalf("RecordJobScratch: %v", err) + } + // Simulate a crash-interrupted completion: the record is mid-lifecycle and + // must be healed back to managed when the deletion aborts. + if err := svc.Store().Update(func(st *StorageState) error { + record := st.Resources[scratchRecordID("o/r", scratchJobDone)] + record.CleanupState = CleanupDeleting + st.Resources[scratchRecordID("o/r", scratchJobDone)] = record + return nil + }); err != nil { + t.Fatalf("seed deleting state: %v", err) + } + // The orphan scratch is a crash-leftover directory with no sidecar record. + report, err := svc.ReconcileJobScratch(context.Background(), true) + if err != nil { + t.Fatalf("ReconcileJobScratch: %v", err) + } + if len(report.ScratchRemoved) != 0 || report.ReclaimedBytes != 0 { + t.Fatalf("revalidated deletions must be aborted: %+v", report) + } + if !contains(report.ScratchKept, scratchJobDone) || !contains(report.ScratchKept, scratchJobOrphan) { + t.Fatalf("reactivated job scratch must be kept: %+v", report) + } + for _, paths := range []JobScratchPaths{donePaths, orphanPaths} { + if _, err := os.Lstat(paths.Root); err != nil { + t.Fatalf("scratch dir %q must survive the revalidation abort: %v", paths.Root, err) + } + } + record, ok := scratchRecord(t, svc, "o/r", scratchJobDone) + if !ok || record.CleanupState != CleanupManaged { + t.Fatalf("record = %+v ok=%v, want healed to managed after the abort", record, ok) + } +} + +// TestReconcileJobScratchDeletionReloadFailureKeepsScratch proves the +// deletion-time reload fails safe: a state read error aborts every deletion +// with a bounded diagnostic instead of deleting on a stale snapshot. +func TestReconcileJobScratchDeletionReloadFailureKeepsScratch(t *testing.T) { + calls := 0 + loader := func(context.Context) (state.RunnerState, error) { + calls++ + if calls == 1 { + return state.NewState(), nil + } + return state.RunnerState{}, errors.New("state store unavailable") + } + svc, root := newRuntimeServiceWithLoader(t, loader) + orphanPaths := prepareScratchWithFile(t, root, scratchJobOrphan, 64) + report, err := svc.ReconcileJobScratch(context.Background(), true) + if err != nil { + t.Fatalf("ReconcileJobScratch: %v", err) + } + if len(report.ScratchRemoved) != 0 || report.ReclaimedBytes != 0 { + t.Fatalf("a reload failure must abort every deletion fail-safe: %+v", report) + } + reloadDiag := false + for _, diagnostic := range report.Diagnostics { + if strings.Contains(diagnostic, "deletion-time state reload") { + reloadDiag = true + } + } + if !reloadDiag { + t.Fatalf("missing deletion-time reload diagnostic: %+v", report.Diagnostics) + } + if _, err := os.Lstat(orphanPaths.Root); err != nil { + t.Fatalf("scratch must survive a deletion-time reload failure: %v", err) + } +} + func TestCompleteJobScratch(t *testing.T) { svc, root := newRuntimeService(t, state.NewState()) paths := prepareScratchWithFile(t, root, scratchJobDone, 16) @@ -410,3 +501,131 @@ func TestEvictRuntimeCachesRejectsForeignHomePath(t *testing.T) { t.Fatalf("tampered home record must produce a diagnostic") } } + +// TestEvictRuntimeCachesSkipsHomesWithActiveJobs proves the in-use guard: a +// home whose repo has any active job keeps its caches (pressured eviction +// must not break in-flight builds), a home with terminal-only jobs is still +// evicted, and when every home is in use the pass reports the deferral. +func TestEvictRuntimeCachesSkipsHomesWithActiveJobs(t *testing.T) { + current := state.NewState() + current.Jobs[scratchJobActive] = state.Job{ID: scratchJobActive, Repo: "o/r", Status: state.StatusRunning} + current.Jobs[scratchJobDone] = state.Job{ID: scratchJobDone, Repo: "o/r2", Status: state.StatusCompleted} + svc, root := newRuntimeServiceWithLoader(t, func(context.Context) (state.RunnerState, error) { return current, nil }) + busyScope := testScope() + idleScope := RuntimeScope{Hostname: "host-1", Repo: "o/r2", Runner: "runner-1"} + homes := map[string]RuntimeHomePaths{} + for _, scope := range []RuntimeScope{busyScope, idleScope} { + paths, err := PrepareRuntimeHome(root, scope) + if err != nil { + t.Fatalf("PrepareRuntimeHome %s: %v", scope.Repo, err) + } + if err := svc.RecordRuntimeHome(context.Background(), scope, paths); err != nil { + t.Fatalf("RecordRuntimeHome %s: %v", scope.Repo, err) + } + writeFile(t, filepath.Join(paths.Home, ".cache", "blob"), 40) + homes[scope.Repo] = paths + } + + report, err := svc.EvictRuntimeCaches(context.Background(), true) + if err != nil { + t.Fatalf("EvictRuntimeCaches: %v", err) + } + busyCache := filepath.Join(homes["o/r"].Home, ".cache") + idleCache := filepath.Join(homes["o/r2"].Home, ".cache") + if contains(report.CacheEvicted, busyCache) { + t.Fatalf("in-use home caches must be preserved: %+v", report) + } + if !contains(report.CacheEvicted, idleCache) { + t.Fatalf("terminal-only home caches must be evicted: %+v", report) + } + if _, err := os.Lstat(filepath.Join(busyCache, "blob")); err != nil { + t.Fatalf("active repo cache must survive eviction: %v", err) + } + if _, err := os.Lstat(idleCache); !os.IsNotExist(err) { + t.Fatalf("idle repo cache must be evicted, err=%v", err) + } + for _, diagnostic := range report.Diagnostics { + if strings.Contains(diagnostic, "sessions are active") { + t.Fatalf("partial skip must not claim full deferral: %+v", report.Diagnostics) + } + } + + // When every recorded home serves an active job the pass defers whole and + // says so. Recreate the evicted idle cache so both homes have content. + writeFile(t, filepath.Join(idleCache, "blob"), 40) + current.Jobs[scratchJobDone] = state.Job{ID: scratchJobDone, Repo: "o/r2", Status: state.StatusRunning} + deferred, err := svc.EvictRuntimeCaches(context.Background(), true) + if err != nil { + t.Fatalf("EvictRuntimeCaches all-active: %v", err) + } + if len(deferred.CacheEvicted) != 0 || deferred.ReclaimedBytes != 0 { + t.Fatalf("all-active eviction must defer every home: %+v", deferred) + } + deferralDiag := false + for _, diagnostic := range deferred.Diagnostics { + if strings.Contains(diagnostic, "eviction deferred") && strings.Contains(diagnostic, "sessions are active") { + deferralDiag = true + } + } + if !deferralDiag { + t.Fatalf("all-active eviction must report the deferral: %+v", deferred.Diagnostics) + } + for repo, paths := range homes { + if _, err := os.Lstat(filepath.Join(paths.Home, ".cache", "blob")); err != nil { + t.Fatalf("all-active eviction must preserve %s caches: %v", repo, err) + } + } +} + +// TestEvictRuntimeCachesRefusesIntermediateSymlink redirects one home's go/ +// subtree at another scope's cache through an intermediate symlink: the +// eviction target itself lstat's as a real directory, so only the +// confinement revalidation can catch the redirection. Eviction must refuse +// and the foreign tree must survive. +func TestEvictRuntimeCachesRefusesIntermediateSymlink(t *testing.T) { + svc, root := newRuntimeService(t, state.NewState()) + scope := testScope() + paths, err := PrepareRuntimeHome(root, scope) + if err != nil { + t.Fatalf("PrepareRuntimeHome: %v", err) + } + if err := svc.RecordRuntimeHome(context.Background(), scope, paths); err != nil { + t.Fatalf("RecordRuntimeHome: %v", err) + } + // A second scope owns a real module cache below the same root. It is never + // recorded, so the eviction pass only reaches it through the symlink. + foreignScope := RuntimeScope{Hostname: "host-1", Repo: "o/r2", Runner: "runner-1"} + foreignPaths, err := PrepareRuntimeHome(root, foreignScope) + if err != nil { + t.Fatalf("PrepareRuntimeHome foreign: %v", err) + } + foreignPayload := filepath.Join(foreignPaths.Home, "go", "pkg", "mod", "m.zip") + writeFile(t, foreignPayload, 40) + if err := os.Symlink(filepath.Join(foreignPaths.Home, "go"), filepath.Join(paths.Home, "go")); err != nil { + t.Fatalf("symlink: %v", err) + } + report, err := svc.EvictRuntimeCaches(context.Background(), true) + if err != nil { + t.Fatalf("EvictRuntimeCaches: %v", err) + } + target := filepath.Join(paths.Home, "go", "pkg", "mod") + if contains(report.CacheEvicted, target) { + t.Fatalf("eviction must refuse an intermediate symlink redirection: %+v", report) + } + redirectionDiag := false + for _, diagnostic := range report.Diagnostics { + if strings.Contains(diagnostic, target) { + redirectionDiag = true + } + } + if !redirectionDiag { + t.Fatalf("redirection must produce a diagnostic naming the target: %+v", report.Diagnostics) + } + info, err := os.Lstat(filepath.Join(paths.Home, "go")) + if err != nil || info.Mode()&os.ModeSymlink == 0 { + t.Fatalf("the intermediate symlink itself must remain, info=%v err=%v", info, err) + } + if _, err := os.Lstat(foreignPayload); err != nil { + t.Fatalf("the foreign scope cache must survive: %v", err) + } +} From 712df9bb95e7d2a65c7c9e5e7a19d048feac8538 Mon Sep 17 00:00:00 2001 From: johnlanni Date: Sun, 9 Aug 2026 00:22:16 +0800 Subject: [PATCH 14/17] fix(runner): make cache eviction handle read-only Go module caches The Go module cache materializes directories 0555 and files 0444, and unlink requires a writable parent, so every removal built on removeOpenedTree failed with permission denied for the non-root runner service user: EvictRuntimeCaches could never evict go/pkg/mod, the largest rebuildable cache. Local root runs masked the bug; the non-root CI runner surfaced it through the acceptance test's TempDir cleanup. - removeOpenedTree now relaxes the validated tree owner-writable (directories 0700, files 0600, only where owner write is missing) through the already-opened root capability before unlinking. Every safety check is unchanged: exact confinement, symlink refusal (the walk never follows links; removal unlinks them via the relaxed parent), and SameFile revalidation before the final unlink. The pass is best-effort: an entry that resists relaxation is left for the removal pass, whose error stays authoritative. - RemoveManagedTree keeps its exact semantics for the legacy session/pool deletion paths; fixing the shared helper also fixes the same latent failure when a retired .sessions/ home holds a read-only Go module cache (and process pools holding read-only git objects), now pinned by a regression test. - TestSharedRuntimeHomeReusesGoCachesAcrossJobs registers a cleanup that relaxes the fixture home tree before t.TempDir removal runs (cleanups are LIFO), so a non-root runner no longer fails cleanup. Audit of the other shared-layout tests (77783bd, 5528b5a): they write only 0600/0700 fixtures or use fakes, so no other test lets the go tool write into a TempDir home. - New regressions: eviction of a 0555/0444 module-cache fixture reclaims every byte, RemoveManagedTree deletes a read-only legacy session runtime, and the relaxation pass is pinned euid-independently (modes relaxed, symlinks and out-of-root targets untouched). Verified as the nobody user: without the fix the regressions and the acceptance TempDir cleanup fail with the exact CI permission-denied signature; with it, the whole storage package passes as non-root. Signed-off-by: johnlanni --- internal/commentrunner/storage/remove.go | 70 ++++++++++- internal/commentrunner/storage/remove_test.go | 113 ++++++++++++++++++ .../storage/runtime_home_acceptance_test.go | 43 +++++++ .../storage/runtime_reconcile_test.go | 45 +++++++ 4 files changed, 270 insertions(+), 1 deletion(-) create mode 100644 internal/commentrunner/storage/remove_test.go diff --git a/internal/commentrunner/storage/remove.go b/internal/commentrunner/storage/remove.go index eb100b1c..3f02ee33 100644 --- a/internal/commentrunner/storage/remove.go +++ b/internal/commentrunner/storage/remove.go @@ -20,7 +20,11 @@ func RemoveManagedTree(workspaceRoot, target, expectedHash string, beforeRemove // removeOpenedTree removes an already validated target directory through an // opened root capability. Callers must prove the target's identity and // confinement first; this helper only guarantees the opened object — not a -// replaced pathname — is what gets removed. +// replaced pathname — is what gets removed. Before any entry is unlinked the +// validated tree is relaxed owner-writable through the same capability: +// tool-managed caches materialize read-only entries (the Go module cache +// writes directories 0555 and files 0444) and unlink requires a writable +// parent, so a non-root service user cannot remove such trees otherwise. func removeOpenedTree(target string, beforeRemove func()) error { info, err := os.Lstat(target) if os.IsNotExist(err) { @@ -44,6 +48,7 @@ func removeOpenedTree(target string, beforeRemove func()) error { if beforeRemove != nil { beforeRemove() } + makeOpenedTreeWritable(targetRoot) entries, err := targetRoot.Open(".") if err != nil { return fmt.Errorf("open deletion target contents: %w", err) @@ -84,3 +89,66 @@ func removeOpenedTree(target string, beforeRemove func()) error { } return nil } + +// makeOpenedTreeWritable relaxes every entry below an opened root so the +// removal pass can unlink it as a non-root owner: directories gain owner rwx +// (0700) and files owner rw (0600), but only where owner write is missing. +// Every access goes through the root capability, so the walk stays bound to +// the proven object even if the target pathname is replaced mid-pass. +// Symlinks are never followed or chmodded; the removal pass unlinks them +// through their relaxed parent directory. The pass is best-effort: an entry +// that resists relaxation (for example one owned by another user) is left +// for the removal pass, whose error remains authoritative. +func makeOpenedTreeWritable(root *os.Root) { + pending := []string{"."} + for len(pending) > 0 { + dir := pending[len(pending)-1] + pending = pending[:len(pending)-1] + info, err := root.Lstat(dir) + if err != nil || !info.IsDir() { + continue + } + if info.Mode().Perm()&0o300 != 0o300 { + // Owner write+execute are needed to unlink children and to list + // the directory, so relax it before descending. + _ = root.Chmod(dir, 0o700) + } + entries, err := readOpenedRootDir(root, dir) + if err != nil { + continue + } + for _, entry := range entries { + switch { + case entry.Type()&os.ModeSymlink != 0: + // Never followed: removal unlinks the entry itself. + case entry.IsDir(): + pending = append(pending, filepath.Join(dir, entry.Name())) + default: + entryInfo, err := entry.Info() + if err != nil { + continue + } + if entryInfo.Mode().Perm()&0o200 == 0 { + _ = root.Chmod(filepath.Join(dir, entry.Name()), 0o600) + } + } + } + } +} + +// readOpenedRootDir lists one directory below an opened root. +func readOpenedRootDir(root *os.Root, dir string) ([]os.DirEntry, error) { + opened, err := root.Open(dir) + if err != nil { + return nil, err + } + entries, readErr := opened.ReadDir(-1) + closeErr := opened.Close() + if readErr != nil { + return nil, readErr + } + if closeErr != nil { + return nil, closeErr + } + return entries, nil +} diff --git a/internal/commentrunner/storage/remove_test.go b/internal/commentrunner/storage/remove_test.go new file mode 100644 index 00000000..31bca06d --- /dev/null +++ b/internal/commentrunner/storage/remove_test.go @@ -0,0 +1,113 @@ +package storage + +import ( + "os" + "path/filepath" + "strings" + "testing" +) + +// makeTreeReadOnly materializes the permission shape the Go module cache +// leaves behind: every directory 0555 and every file 0444. +func makeTreeReadOnly(t *testing.T, root string) { + t.Helper() + if err := filepath.WalkDir(root, func(path string, entry os.DirEntry, err error) error { + if err != nil { + return err + } + if entry.Type()&os.ModeSymlink != 0 { + return nil + } + if entry.IsDir() { + return os.Chmod(path, 0o555) + } + return os.Chmod(path, 0o444) + }); err != nil { + t.Fatalf("make %s read-only: %v", root, err) + } +} + +// TestMakeOpenedTreeWritableRelaxesReadOnlyTree pins the relaxation pass +// itself, independent of euid: directories become 0700 and files 0600 so a +// non-root owner can unlink the tree, while symlinks are never followed and +// nothing outside the opened root changes mode. +func TestMakeOpenedTreeWritableRelaxesReadOnlyTree(t *testing.T) { + base := t.TempDir() + t.Cleanup(func() { relaxTreeForCleanup(t, base) }) + tree := filepath.Join(base, "tree") + outside := filepath.Join(base, "outside") + writeFile(t, filepath.Join(tree, "sub", "nested", "ro.txt"), 8) + writeFile(t, filepath.Join(tree, "ro.txt"), 8) + writeFile(t, filepath.Join(outside, "target.txt"), 8) + if err := os.Symlink(filepath.Join(outside, "target.txt"), filepath.Join(tree, "link")); err != nil { + t.Fatalf("symlink: %v", err) + } + makeTreeReadOnly(t, tree) + makeTreeReadOnly(t, outside) + + opened, err := os.OpenRoot(tree) + if err != nil { + t.Fatalf("open root: %v", err) + } + makeOpenedTreeWritable(opened) + if err := opened.Close(); err != nil { + t.Fatalf("close root: %v", err) + } + + for path, want := range map[string]os.FileMode{ + tree: 0o700, + filepath.Join(tree, "sub"): 0o700, + filepath.Join(tree, "sub", "nested"): 0o700, + filepath.Join(tree, "sub", "nested", "ro.txt"): 0o600, + filepath.Join(tree, "ro.txt"): 0o600, + } { + info, err := os.Lstat(path) + if err != nil { + t.Fatalf("lstat %s: %v", path, err) + } + if got := info.Mode().Perm(); got != want { + t.Fatalf("mode %s = %o, want %o", path, got, want) + } + } + // The symlink is never followed: it survives as a link and its target + // outside the opened root keeps its read-only mode. + linkInfo, err := os.Lstat(filepath.Join(tree, "link")) + if err != nil || linkInfo.Mode()&os.ModeSymlink == 0 { + t.Fatalf("link must remain a symlink: info=%v err=%v", linkInfo, err) + } + targetInfo, err := os.Lstat(filepath.Join(outside, "target.txt")) + if err != nil { + t.Fatalf("lstat outside target: %v", err) + } + if got := targetInfo.Mode().Perm(); got != 0o444 { + t.Fatalf("outside target mode = %o, want 444 (never relaxed)", got) + } + outsideInfo, err := os.Lstat(outside) + if err != nil { + t.Fatalf("lstat outside dir: %v", err) + } + if got := outsideInfo.Mode().Perm(); got != 0o555 { + t.Fatalf("outside dir mode = %o, want 555 (never relaxed)", got) + } +} + +// TestRemoveManagedTreeRemovesReadOnlySessionRuntime is the legacy-layout +// regression: a retired .sessions/ runtime whose home holds a +// read-only Go module cache must still delete cleanly for the non-root +// service user. The shared removal helper relaxes the validated tree through +// the opened capability before unlinking, with every validation unchanged. +func TestRemoveManagedTreeRemovesReadOnlySessionRuntime(t *testing.T) { + root := testRoot(t) + t.Cleanup(func() { relaxTreeForCleanup(t, root) }) + hash := strings.Repeat("ab", 16) + target := filepath.Join(root, SessionsDirName, hash) + writeFile(t, filepath.Join(target, "home", "go", "pkg", "mod", "example.com", "dep@v1.0.0", "dep.go"), 24) + makeTreeReadOnly(t, filepath.Join(target, "home")) + + if err := RemoveManagedTree(root, target, hash, nil); err != nil { + t.Fatalf("RemoveManagedTree on a read-only runtime home: %v", err) + } + if _, err := os.Lstat(target); !os.IsNotExist(err) { + t.Fatalf("session runtime must be gone, err=%v", err) + } +} diff --git a/internal/commentrunner/storage/runtime_home_acceptance_test.go b/internal/commentrunner/storage/runtime_home_acceptance_test.go index 579139d7..1a914a19 100644 --- a/internal/commentrunner/storage/runtime_home_acceptance_test.go +++ b/internal/commentrunner/storage/runtime_home_acceptance_test.go @@ -29,6 +29,13 @@ func TestSharedRuntimeHomeReusesGoCachesAcrossJobs(t *testing.T) { } root := t.TempDir() + // The real go tool leaves the module cache below the shared home + // read-only (directories 0555, files 0444) and unlink requires a + // writable parent, so a non-root runner fails t.TempDir removal with + // permission denied. Relax the fixture tree first: cleanups run LIFO + // and root's TempDir cleanup was registered above, so this runs before + // it. + t.Cleanup(func() { relaxTreeForCleanup(t, root) }) scope := RuntimeScope{Hostname: "github.com", Repo: "o/r", Runner: "acceptance"} home, err := PrepareRuntimeHome(root, scope) if err != nil { @@ -270,6 +277,42 @@ func countRegularFiles(t *testing.T, root string) int { return count } +// relaxTreeForCleanup makes every entry below root owner-writable — 0700 for +// directories, 0600 for files — so t.TempDir removal cannot fail on +// read-only tool-managed trees (the Go module cache materializes directories +// 0555 and files 0444, and unlink requires a writable parent) when the test +// runs as a non-root user. Symlinks are skipped: their removal only needs +// the relaxed parent. The walk is top-down, so a directory is relaxed before +// it is descended into. Individual failures are logged, not fatal: a +// leftover read-only entry still surfaces as the t.TempDir cleanup error. +func relaxTreeForCleanup(t *testing.T, root string) { + t.Helper() + _ = filepath.WalkDir(root, func(path string, entry os.DirEntry, err error) error { + if err != nil { + t.Logf("relax %s: %v", path, err) + return nil + } + if entry.Type()&os.ModeSymlink != 0 { + return nil + } + mode := os.FileMode(0o600) + if entry.IsDir() { + mode = 0o700 + } + info, infoErr := entry.Info() + if infoErr != nil { + t.Logf("relax %s: %v", path, infoErr) + return nil + } + if info.Mode().Perm()&0o200 == 0 { + if chmodErr := os.Chmod(path, mode); chmodErr != nil { + t.Logf("relax %s: %v", path, chmodErr) + } + } + return nil + }) +} + // TestPrepareRuntimeHomeAndScratchNeverTouchForeignRoots is the fast unit-level // cutover acceptance: prepare, reconcile, and evict against a fresh root while // a pre-existing old-style root (with a `.sessions//home` fixture) sits diff --git a/internal/commentrunner/storage/runtime_reconcile_test.go b/internal/commentrunner/storage/runtime_reconcile_test.go index 3f98e8e1..ee4f8fac 100644 --- a/internal/commentrunner/storage/runtime_reconcile_test.go +++ b/internal/commentrunner/storage/runtime_reconcile_test.go @@ -432,6 +432,51 @@ func TestEvictRuntimeCaches(t *testing.T) { } } +// TestEvictRuntimeCachesRemovesReadOnlyGoModuleCache is the non-root-runner +// regression for the CI failure mode: the Go module cache materializes +// directories 0555 and files 0444, and unlink requires a writable parent, so +// a plain removal fails for the service user with permission denied. +// Eviction must relax the validated tree through the opened root capability +// and still reclaim every byte. As root the removal would succeed either +// way, so the relaxation pass itself is pinned euid-independently by +// TestMakeOpenedTreeWritableRelaxesReadOnlyTree. +func TestEvictRuntimeCachesRemovesReadOnlyGoModuleCache(t *testing.T) { + svc, root := newRuntimeService(t, state.NewState()) + // If eviction regresses, the read-only fixture must not break t.TempDir + // removal as a non-root user and mask the primary failure. + t.Cleanup(func() { relaxTreeForCleanup(t, root) }) + scope := testScope() + paths, err := PrepareRuntimeHome(root, scope) + if err != nil { + t.Fatalf("PrepareRuntimeHome: %v", err) + } + if err := svc.RecordRuntimeHome(context.Background(), scope, paths); err != nil { + t.Fatalf("RecordRuntimeHome: %v", err) + } + modCache := filepath.Join(paths.Home, "go", "pkg", "mod") + writeFile(t, filepath.Join(modCache, "example.com", "dep@v1.0.0", "dep.go"), 48) + writeFile(t, filepath.Join(modCache, "cache", "download", "example.com", "dep", "@v", "v1.0.0.zip"), 16) + // Leave the cache exactly the way the go tool does: read-only. + makeTreeReadOnly(t, modCache) + + report, err := svc.EvictRuntimeCaches(context.Background(), true) + if err != nil { + t.Fatalf("EvictRuntimeCaches: %v", err) + } + if !contains(report.CacheEvicted, modCache) { + t.Fatalf("read-only module cache must be evicted: %+v", report) + } + if report.ReclaimedBytes != 64 { + t.Fatalf("reclaimed = %d, want 64", report.ReclaimedBytes) + } + if _, err := os.Lstat(modCache); !os.IsNotExist(err) { + t.Fatalf("read-only module cache must be gone, err=%v", err) + } + if _, err := os.Lstat(filepath.Join(paths.Root, "scope.json")); err != nil { + t.Fatalf("protected scope binding must survive eviction: %v", err) + } +} + func TestEvictRuntimeCachesNeverFollowsSymlinks(t *testing.T) { svc, root := newRuntimeService(t, state.NewState()) scope := testScope() From 1dbcf1b259e9c7dd6375cf97a93c599475e03bcb Mon Sep 17 00:00:00 2001 From: johnlanni Date: Sun, 9 Aug 2026 00:52:52 +0800 Subject: [PATCH 15/17] fix(runner): close eviction revalidation and dirent-type edges MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two review P2s in the shared-home cleanup paths: - makeOpenedTreeWritable trusted the dirent file/dir classification even when a filesystem reports DT_UNKNOWN (some FUSE/NFS/CIFS mounts), so a subdirectory could be chmodded 0600 as a file and strand its children; unknown-type entries are now reclassified through the opened root capability. Directories are also relaxed whenever any owner rwx bit is missing (0o700 predicate, not 0o300): a 0300 directory is writable yet unlistable, so the descent never reached its children. - EvictRuntimeCaches classified homes once at pass start and then removed every cache on that stale snapshot. Each home's first cache removal now revalidates its repo against freshly loaded runner state, mirroring revalidateScratchDeletion: a repo that turned active mid-pass — or a reload failure — skips the home fail-safe with a bounded diagnostic. Signed-off-by: johnlanni --- internal/commentrunner/storage/remove.go | 78 +++++++--- internal/commentrunner/storage/remove_test.go | 134 ++++++++++++++++++ .../storage/runtime_reconcile.go | 41 ++++++ .../storage/runtime_reconcile_test.go | 104 ++++++++++++++ 4 files changed, 339 insertions(+), 18 deletions(-) diff --git a/internal/commentrunner/storage/remove.go b/internal/commentrunner/storage/remove.go index 3f02ee33..e9f0dfcf 100644 --- a/internal/commentrunner/storage/remove.go +++ b/internal/commentrunner/storage/remove.go @@ -92,7 +92,9 @@ func removeOpenedTree(target string, beforeRemove func()) error { // makeOpenedTreeWritable relaxes every entry below an opened root so the // removal pass can unlink it as a non-root owner: directories gain owner rwx -// (0700) and files owner rw (0600), but only where owner write is missing. +// (0700) whenever any owner bit is missing — write+execute unlink children, +// read lists the directory, so a 0300 directory is writable yet undescendable +// — and files gain owner rw (0600), but only where owner write is missing. // Every access goes through the root capability, so the walk stays bound to // the proven object even if the target pathname is replaced mid-pass. // Symlinks are never followed or chmodded; the removal pass unlinks them @@ -108,9 +110,10 @@ func makeOpenedTreeWritable(root *os.Root) { if err != nil || !info.IsDir() { continue } - if info.Mode().Perm()&0o300 != 0o300 { - // Owner write+execute are needed to unlink children and to list - // the directory, so relax it before descending. + if info.Mode().Perm()&0o700 != 0o700 { + // Owner write+execute are needed to unlink children and owner read + // to list the directory, so relax on any missing owner bit before + // descending. _ = root.Chmod(dir, 0o700) } entries, err := readOpenedRootDir(root, dir) @@ -118,24 +121,63 @@ func makeOpenedTreeWritable(root *os.Root) { continue } for _, entry := range entries { - switch { - case entry.Type()&os.ModeSymlink != 0: - // Never followed: removal unlinks the entry itself. - case entry.IsDir(): - pending = append(pending, filepath.Join(dir, entry.Name())) - default: - entryInfo, err := entry.Info() - if err != nil { - continue - } - if entryInfo.Mode().Perm()&0o200 == 0 { - _ = root.Chmod(filepath.Join(dir, entry.Name()), 0o600) - } - } + pending = relaxOpenedTreeEntry(root, dir, entry, pending) } } } +// relaxOpenedTreeEntry classifies one directory entry below the opened root +// and returns the updated descent queue: subdirectories are queued so their +// own pop relaxes them before they are listed, plain files missing owner +// write are relaxed in place, and symlinks are never followed or chmodded. +// An entry whose dirent carries no type (DT_UNKNOWN on some FUSE/NFS/CIFS +// mounts) lstat's through the same root capability instead of trusting the +// file fallback: a subdirectory misclassified as a file would be chmodded +// 0600 and strand its children behind an undescendable directory. +func relaxOpenedTreeEntry(root *os.Root, dir string, entry os.DirEntry, pending []string) []string { + child := filepath.Join(dir, entry.Name()) + entryType := entry.Type() + if entryType&os.ModeSymlink != 0 { + // Never followed: removal unlinks the entry itself. + return pending + } + if entryType.IsDir() { + return append(pending, child) + } + if entryType != 0 { + // Known non-directory type: relax the entry when owner write is missing. + entryInfo, err := entry.Info() + if err != nil { + return pending + } + relaxOpenedFile(root, child, entryInfo.Mode()) + return pending + } + // The dirent type is unknown: classify through the capability. + entryInfo, err := root.Lstat(child) + if err != nil { + return pending + } + mode := entryInfo.Mode() + if mode&os.ModeSymlink != 0 { + // A symlink behind an unknown dirent type is left alone. + return pending + } + if mode.IsDir() { + return append(pending, child) + } + relaxOpenedFile(root, child, mode) + return pending +} + +// relaxOpenedFile relaxes one non-directory entry below the opened root when +// owner write is missing, so the removal pass can unlink it. +func relaxOpenedFile(root *os.Root, path string, mode os.FileMode) { + if mode.Perm()&0o200 == 0 { + _ = root.Chmod(path, 0o600) + } +} + // readOpenedRootDir lists one directory below an opened root. func readOpenedRootDir(root *os.Root, dir string) ([]os.DirEntry, error) { opened, err := root.Open(dir) diff --git a/internal/commentrunner/storage/remove_test.go b/internal/commentrunner/storage/remove_test.go index 31bca06d..b38fdd6d 100644 --- a/internal/commentrunner/storage/remove_test.go +++ b/internal/commentrunner/storage/remove_test.go @@ -111,3 +111,137 @@ func TestRemoveManagedTreeRemovesReadOnlySessionRuntime(t *testing.T) { t.Fatalf("session runtime must be gone, err=%v", err) } } + +func chmod(t *testing.T, path string, mode os.FileMode) { + t.Helper() + if err := os.Chmod(path, mode); err != nil { + t.Fatalf("chmod %s to %o: %v", path, mode, err) + } +} + +func assertMode(t *testing.T, path string, want os.FileMode) { + t.Helper() + info, err := os.Lstat(path) + if err != nil { + t.Fatalf("lstat %s: %v", path, err) + } + if got := info.Mode().Perm(); got != want { + t.Fatalf("mode %s = %o, want %o", path, got, want) + } +} + +// unknownTypeEntry simulates a directory entry read from a filesystem that +// reports DT_UNKNOWN (some FUSE/NFS/CIFS mounts): the dirent carries no type +// bits, and IsDir — derived from those bits — is false even for directories. +type unknownTypeEntry struct { + name string + path string +} + +func (e unknownTypeEntry) Name() string { return e.name } +func (e unknownTypeEntry) IsDir() bool { return e.Type().IsDir() } +func (e unknownTypeEntry) Type() os.FileMode { return 0 } +func (e unknownTypeEntry) Info() (os.FileInfo, error) { return os.Lstat(e.path) } + +// TestRelaxOpenedTreeEntryClassifiesUnknownDirentType pins the DT_UNKNOWN +// fallback: an entry whose dirent carries no type must be classified through +// the opened capability, never trusted as a file. A subdirectory behind an +// unknown type is enqueued for its own relaxation pass — and not chmodded +// 0600, which would strand its children — a read-only file is relaxed, and a +// symlink is left alone. +func TestRelaxOpenedTreeEntryClassifiesUnknownDirentType(t *testing.T) { + base := t.TempDir() + t.Cleanup(func() { relaxTreeForCleanup(t, base) }) + tree := filepath.Join(base, "tree") + outside := filepath.Join(base, "outside") + writeFile(t, filepath.Join(tree, "subdir", "child.txt"), 8) + writeFile(t, filepath.Join(tree, "plain.txt"), 8) + writeFile(t, filepath.Join(outside, "target.txt"), 8) + if err := os.Symlink(filepath.Join(outside, "target.txt"), filepath.Join(tree, "link")); err != nil { + t.Fatalf("symlink: %v", err) + } + chmod(t, filepath.Join(tree, "subdir", "child.txt"), 0o444) + chmod(t, filepath.Join(tree, "subdir"), 0o555) + chmod(t, filepath.Join(tree, "plain.txt"), 0o444) + chmod(t, filepath.Join(outside, "target.txt"), 0o444) + + opened, err := os.OpenRoot(tree) + if err != nil { + t.Fatalf("open root: %v", err) + } + pending := relaxOpenedTreeEntry(opened, ".", unknownTypeEntry{name: "subdir", path: filepath.Join(tree, "subdir")}, nil) + if len(pending) != 1 || pending[0] != "subdir" { + t.Fatalf("unknown-type subdirectory must be enqueued for descent, pending=%v", pending) + } + // Enqueueing must not have relaxed the directory as if it were a file. + assertMode(t, filepath.Join(tree, "subdir"), 0o555) + pending = relaxOpenedTreeEntry(opened, ".", unknownTypeEntry{name: "plain.txt", path: filepath.Join(tree, "plain.txt")}, pending) + if len(pending) != 1 { + t.Fatalf("a file must not be enqueued, pending=%v", pending) + } + assertMode(t, filepath.Join(tree, "plain.txt"), 0o600) + pending = relaxOpenedTreeEntry(opened, ".", unknownTypeEntry{name: "link", path: filepath.Join(tree, "link")}, pending) + if len(pending) != 1 { + t.Fatalf("a symlink must not be enqueued, pending=%v", pending) + } + if err := opened.Close(); err != nil { + t.Fatalf("close root: %v", err) + } + linkInfo, err := os.Lstat(filepath.Join(tree, "link")) + if err != nil || linkInfo.Mode()&os.ModeSymlink == 0 { + t.Fatalf("link must remain a symlink: info=%v err=%v", linkInfo, err) + } + assertMode(t, filepath.Join(outside, "target.txt"), 0o444) +} + +// TestMakeOpenedTreeWritableRelaxesUnlistableDirectory pins the owner-read +// edge of the directory predicate: a 0300 directory (owner write+execute, no +// read) can have children unlinked by name but cannot be listed, and a 0500 +// directory cannot be written into. Both must be relaxed to 0700 before the +// descent so their children are relaxed and the removal pass can unlink the +// whole tree. The mode assertions hold under any euid; the removal assertion +// is what fails for a non-root owner when the relaxation regresses. +func TestMakeOpenedTreeWritableRelaxesUnlistableDirectory(t *testing.T) { + base := t.TempDir() + t.Cleanup(func() { relaxTreeForCleanup(t, base) }) + tree := filepath.Join(base, "tree") + writeFile(t, filepath.Join(tree, "wx", "nested", "deep.txt"), 8) + writeFile(t, filepath.Join(tree, "wx", "child.txt"), 8) + writeFile(t, filepath.Join(tree, "rx", "child.txt"), 8) + // Chmod children before parents: once a directory drops owner read or + // write, changing entries below it is only possible by traversal. + chmod(t, filepath.Join(tree, "wx", "nested", "deep.txt"), 0o444) + chmod(t, filepath.Join(tree, "wx", "child.txt"), 0o444) + chmod(t, filepath.Join(tree, "wx", "nested"), 0o555) + chmod(t, filepath.Join(tree, "wx"), 0o300) + chmod(t, filepath.Join(tree, "rx", "child.txt"), 0o444) + chmod(t, filepath.Join(tree, "rx"), 0o500) + + opened, err := os.OpenRoot(tree) + if err != nil { + t.Fatalf("open root: %v", err) + } + makeOpenedTreeWritable(opened) + if err := opened.Close(); err != nil { + t.Fatalf("close root: %v", err) + } + + for path, want := range map[string]os.FileMode{ + filepath.Join(tree, "wx"): 0o700, + filepath.Join(tree, "wx", "nested"): 0o700, + filepath.Join(tree, "wx", "nested", "deep.txt"): 0o600, + filepath.Join(tree, "wx", "child.txt"): 0o600, + filepath.Join(tree, "rx"): 0o700, + filepath.Join(tree, "rx", "child.txt"): 0o600, + } { + assertMode(t, path, want) + } + + // The relaxed tree removes cleanly through the capability. + if err := removeOpenedTree(tree, nil); err != nil { + t.Fatalf("removeOpenedTree on an unlistable-directory tree: %v", err) + } + if _, err := os.Lstat(tree); !os.IsNotExist(err) { + t.Fatalf("tree must be gone, err=%v", err) + } +} diff --git a/internal/commentrunner/storage/runtime_reconcile.go b/internal/commentrunner/storage/runtime_reconcile.go index 37a98e38..7a4f6786 100644 --- a/internal/commentrunner/storage/runtime_reconcile.go +++ b/internal/commentrunner/storage/runtime_reconcile.go @@ -267,6 +267,26 @@ func (s *Service) revalidateScratchDeletion(ctx context.Context, jobID string, w } } +// revalidateHomeEviction reloads runner state immediately before a home's +// first cache removal, mirroring revalidateScratchDeletion's deletion-time +// discipline: a repo with any job that turned active since the pass snapshot +// aborts the home's eviction this pass. A reload failure aborts the eviction +// fail-safe: the caches survive and are retried on a later pass. +func (s *Service) revalidateHomeEviction(ctx context.Context, repo string) (bool, error) { + fresh, err := s.stateLoader(ctx) + if err != nil { + return true, fmt.Errorf("deletion-time state reload: %w", err) + } + fresh.Normalize() + repo = strings.TrimSpace(repo) + for _, job := range fresh.Jobs { + if jobRuntimeActive(job.Status) && strings.TrimSpace(job.Repo) == repo { + return true, nil + } + } + return false, nil +} + // healScratchRecord returns a live job's scratch record to managed state: a // crash-interrupted completion raced a still-active job, so the record is // healed instead of deleting live scratch. @@ -319,6 +339,10 @@ func (s *Service) removeJobScratch(record PhysicalResource, report *RuntimeRecon // pressured eviction must not break in-flight builds. When every recorded // home is skipped for that reason the pass reports the deferral so an // operator (or the pressured-admission path) can tell eviction did run. +// Every home's first cache removal revalidates its repo against freshly +// loaded runner state, mirroring the scratch pass's deletion-time discipline: +// a repo that turned active since the pass snapshot — or a reload failure — +// skips the home fail-safe. func (s *Service) EvictRuntimeCaches(ctx context.Context, apply bool) (RuntimeReconcileReport, error) { _, release, err := EnsureOwner(ctx, s.root) if err != nil { @@ -368,6 +392,7 @@ func (s *Service) EvictRuntimeCaches(ctx context.Context, apply bool) (RuntimeRe continue } homeDir := RuntimeHomePathsFor(home.Path).Home + homeRevalidated := false for _, cacheDir := range RuntimeCacheDirs(homeDir) { info, statErr := os.Lstat(cacheDir) if errors.Is(statErr, os.ErrNotExist) { @@ -386,6 +411,22 @@ func (s *Service) EvictRuntimeCaches(ctx context.Context, apply bool) (RuntimeRe report.Diagnostics = append(report.Diagnostics, fmt.Sprintf("would evict runtime cache %s (%d bytes)", cacheDir, measured)) continue } + if !homeRevalidated { + // Deletion-time revalidation, once per home immediately before + // its first cache removal (mirroring revalidateScratchDeletion): + // a repo that turned active since the pass snapshot — or a + // reload failure — skips the home fail-safe. + homeRevalidated = true + abort, revalidateErr := s.revalidateHomeEviction(ctx, home.Repo) + if revalidateErr != nil { + report.Diagnostics = append(report.Diagnostics, safeDiagnostic("runtime home "+home.PhysicalHash+" cache eviction skipped: "+revalidateErr.Error())) + break + } + if abort { + report.Diagnostics = append(report.Diagnostics, safeDiagnostic("runtime home "+home.PhysicalHash+" cache eviction skipped: repo "+strings.TrimSpace(home.Repo)+" turned active during the pass")) + break + } + } if err := removeOpenedTree(cacheDir, nil); err != nil { report.Diagnostics = append(report.Diagnostics, safeDiagnostic("runtime cache "+cacheDir+" removal failed: "+err.Error())) continue diff --git a/internal/commentrunner/storage/runtime_reconcile_test.go b/internal/commentrunner/storage/runtime_reconcile_test.go index ee4f8fac..952629be 100644 --- a/internal/commentrunner/storage/runtime_reconcile_test.go +++ b/internal/commentrunner/storage/runtime_reconcile_test.go @@ -674,3 +674,107 @@ func TestEvictRuntimeCachesRefusesIntermediateSymlink(t *testing.T) { t.Fatalf("the foreign scope cache must survive: %v", err) } } + +// TestEvictRuntimeCachesRevalidatesBeforeFirstRemoval mirrors the scratch +// pass's deletion-time revalidation with the engine's flip-loader pattern: +// the pass snapshot classifies both homes as eviction-eligible, but the +// deletion-time reload sees one home's repo running again, so that home is +// skipped whole — no cache of its is removed — while the still-idle home is +// evicted. +func TestEvictRuntimeCachesRevalidatesBeforeFirstRemoval(t *testing.T) { + before := state.NewState() + after := state.NewState() + after.Jobs[scratchJobActive] = state.Job{ID: scratchJobActive, Repo: "o/r", Status: state.StatusRunning} + svc, root := newRuntimeServiceWithLoader(t, flipLoader(before, after)) + idleScope := RuntimeScope{Hostname: "host-1", Repo: "o/r2", Runner: "runner-1"} + homes := map[string]RuntimeHomePaths{} + for _, scope := range []RuntimeScope{testScope(), idleScope} { + paths, err := PrepareRuntimeHome(root, scope) + if err != nil { + t.Fatalf("PrepareRuntimeHome %s: %v", scope.Repo, err) + } + if err := svc.RecordRuntimeHome(context.Background(), scope, paths); err != nil { + t.Fatalf("RecordRuntimeHome %s: %v", scope.Repo, err) + } + writeFile(t, filepath.Join(paths.Home, ".cache", "blob"), 40) + writeFile(t, filepath.Join(paths.Home, "go", "pkg", "mod", "m.zip"), 24) + homes[scope.Repo] = paths + } + + report, err := svc.EvictRuntimeCaches(context.Background(), true) + if err != nil { + t.Fatalf("EvictRuntimeCaches: %v", err) + } + busyHome := homes["o/r"].Home + idleHome := homes["o/r2"].Home + for _, rel := range []string{filepath.Join(".cache", "blob"), filepath.Join("go", "pkg", "mod", "m.zip")} { + if _, err := os.Lstat(filepath.Join(busyHome, rel)); err != nil { + t.Fatalf("reactivated repo cache %q must survive eviction: %v", rel, err) + } + if _, err := os.Lstat(filepath.Join(idleHome, rel)); !os.IsNotExist(err) { + t.Fatalf("idle repo cache %q must be evicted, err=%v", rel, err) + } + } + for _, evicted := range report.CacheEvicted { + if strings.HasPrefix(evicted, busyHome+string(os.PathSeparator)) { + t.Fatalf("reactivated repo home must not appear in CacheEvicted: %+v", report) + } + } + if !contains(report.CacheEvicted, filepath.Join(idleHome, ".cache")) || + !contains(report.CacheEvicted, filepath.Join(idleHome, "go", "pkg", "mod")) { + t.Fatalf("idle home caches must be evicted: %+v", report) + } + skipDiag := false + for _, diagnostic := range report.Diagnostics { + if strings.Contains(diagnostic, "cache eviction skipped") && strings.Contains(diagnostic, "repo o/r turned active") { + skipDiag = true + } + } + if !skipDiag { + t.Fatalf("revalidation abort must produce a skip diagnostic: %+v", report.Diagnostics) + } +} + +// TestEvictRuntimeCachesDeletionReloadFailureKeepsCaches proves the +// deletion-time reload fails safe: a state read error immediately before a +// home's first cache removal skips the home with a bounded diagnostic +// instead of evicting on a stale snapshot. +func TestEvictRuntimeCachesDeletionReloadFailureKeepsCaches(t *testing.T) { + calls := 0 + loader := func(context.Context) (state.RunnerState, error) { + calls++ + if calls == 1 { + return state.NewState(), nil + } + return state.RunnerState{}, errors.New("state store unavailable") + } + svc, root := newRuntimeServiceWithLoader(t, loader) + paths, err := PrepareRuntimeHome(root, testScope()) + if err != nil { + t.Fatalf("PrepareRuntimeHome: %v", err) + } + if err := svc.RecordRuntimeHome(context.Background(), testScope(), paths); err != nil { + t.Fatalf("RecordRuntimeHome: %v", err) + } + writeFile(t, filepath.Join(paths.Home, ".cache", "blob"), 40) + + report, err := svc.EvictRuntimeCaches(context.Background(), true) + if err != nil { + t.Fatalf("EvictRuntimeCaches: %v", err) + } + if len(report.CacheEvicted) != 0 || report.ReclaimedBytes != 0 { + t.Fatalf("a reload failure must skip the home fail-safe: %+v", report) + } + reloadDiag := false + for _, diagnostic := range report.Diagnostics { + if strings.Contains(diagnostic, "deletion-time state reload") { + reloadDiag = true + } + } + if !reloadDiag { + t.Fatalf("missing deletion-time reload diagnostic: %+v", report.Diagnostics) + } + if _, err := os.Lstat(filepath.Join(paths.Home, ".cache", "blob")); err != nil { + t.Fatalf("caches must survive a deletion-time reload failure: %v", err) + } +} From 5d3b8589b309999204243710058b1de2740bf91f Mon Sep 17 00:00:00 2001 From: johnlanni Date: Sun, 9 Aug 2026 11:31:03 +0800 Subject: [PATCH 16/17] fix(runner): merge acpx agent overrides per-agent under lock in the shared HOME The runner-scoped shared TempHome made per-dispatch whole-file writes to .acpx/config.json race: a dispatch for an agent with no host override removed the whole file, and MaterializeAgentOverride replaced it with a single-agent document, so concurrent dispatches in one runner scope silently lost each other's overrides and acpx fell back to the builtin adapter. Add acpx.ApplyAgentOverride: a per-agent upsert/delete applied under an in-process keyed mutex plus a bounded flock on config.json.lock (~2s retry, then a clear error). The read-modify-write preserves all unrelated top-level fields and peer agents' raw entries, fails closed (no modification) on malformed, oversized, symlinked, or non-regular configs, deletes the file only when it existed solely for overrides, and installs the result with a temp file plus rename so readers never observe partial JSON. MaterializeAgentOverride is reimplemented on top of it so no caller can whole-file clobber, and materializeHostAcpxAgentOverride now prunes only the refreshed agent's entry when the host has no override for it. Signed-off-by: johnlanni --- internal/acpx/agent_override.go | 216 +++++++- internal/acpx/agent_override_test.go | 481 ++++++++++++++++++ internal/acpx/filelock_unix.go | 21 + internal/acpx/filelock_windows.go | 26 + internal/commentrunner/jobs/dispatcher.go | 13 +- .../commentrunner/jobs/dispatcher_test.go | 81 +++ 6 files changed, 806 insertions(+), 32 deletions(-) create mode 100644 internal/acpx/filelock_unix.go create mode 100644 internal/acpx/filelock_windows.go diff --git a/internal/acpx/agent_override.go b/internal/acpx/agent_override.go index cad51c85..e6f85b75 100644 --- a/internal/acpx/agent_override.go +++ b/internal/acpx/agent_override.go @@ -10,12 +10,25 @@ import ( "path/filepath" "regexp" "strings" + "sync" + "time" ) const maxAgentConfigBytes = 1 << 20 +// overrideLockTimeout bounds how long ApplyAgentOverride waits for the +// cross-process config lock before failing the dispatch with a clear error. +// The lock is only ever held for one read-modify-write of a <=1MiB file, so +// two seconds is generous. +const overrideLockTimeout = 2 * time.Second + var codexAdapterPattern = regexp.MustCompile(`@agentclientprotocol/codex-acp@[A-Za-z0-9._~^+\-]+`) +// overrideWriteLocks serializes in-process writers per config file. The flock +// on config.json.lock additionally serializes cooperating processes (e.g. a +// runner and an operator CLI sharing one HOME). +var overrideWriteLocks sync.Map // map[string]*sync.Mutex, keyed by cleaned lock path + type AgentOverride struct { Agent string Command string @@ -64,45 +77,189 @@ func LoadAgentOverride(home, agent string) (AgentOverride, bool, error) { if command == "" { return AgentOverride{}, false, nil } + if err := validateOverrideCommand(agent, command, selected.Args); err != nil { + return AgentOverride{}, false, err + } + args := make([]string, len(selected.Args)) + copy(args, selected.Args) + return AgentOverride{Agent: agent, Command: command, Args: args, Source: path}, true, nil +} + +// validateOverrideCommand enforces the exact bounds LoadAgentOverride enforces +// when reading an override back, so a value written through ApplyAgentOverride +// can never be rejected by the reader. +func validateOverrideCommand(agent, command string, args []string) error { if len(command) > 4096 || strings.ContainsAny(command, "\x00\r\n") { - return AgentOverride{}, false, fmt.Errorf("acpx %s agent override is invalid", agent) + return fmt.Errorf("acpx %s agent override is invalid", agent) } - if len(selected.Args) > 64 { - return AgentOverride{}, false, fmt.Errorf("acpx %s agent override has too many arguments", agent) + if len(args) > 64 { + return fmt.Errorf("acpx %s agent override has too many arguments", agent) } - args := make([]string, len(selected.Args)) total := 0 - for index, arg := range selected.Args { + for _, arg := range args { total += len(arg) if len(arg) > 4096 || total > 16*1024 || strings.ContainsAny(arg, "\x00\r\n") { - return AgentOverride{}, false, fmt.Errorf("acpx %s agent override arguments are invalid", agent) + return fmt.Errorf("acpx %s agent override arguments are invalid", agent) } - args[index] = arg } - return AgentOverride{Agent: agent, Command: command, Args: args, Source: path}, true, nil + return nil } -func MaterializeAgentOverride(home string, override AgentOverride) error { - if strings.TrimSpace(home) == "" || strings.TrimSpace(override.Agent) == "" || strings.TrimSpace(override.Command) == "" { - return fmt.Errorf("acpx agent override destination and value are required") +// ApplyAgentOverride merges one agent's override into +// /.acpx/config.json under an in-process keyed mutex plus a bounded +// flock on config.json.lock, so concurrent dispatches sharing one runtime +// HOME never clobber each other's per-agent entries. A nil override deletes +// only that agent's entry; when no agents and no other top-level keys remain +// the file existed solely for overrides and is removed. Every other top-level +// field and every other agent's raw entry is preserved. The lock is held only +// for this single read-modify-write, and the result is installed with a temp +// file plus rename, so readers never observe a partial document. A malformed, +// oversized, symlinked, or otherwise non-regular config fails closed: the +// error is returned and the file is left untouched. +func ApplyAgentOverride(home, agent string, override *AgentOverride) error { + agent = strings.TrimSpace(agent) + if strings.TrimSpace(home) == "" || agent == "" { + return fmt.Errorf("acpx agent override destination and agent are required") } dir := filepath.Join(filepath.Clean(home), ".acpx") + target := filepath.Join(dir, "config.json") + if override != nil { + if name := strings.TrimSpace(override.Agent); name != "" && name != agent { + return fmt.Errorf("acpx %s agent override does not match agent %q", name, agent) + } + if strings.TrimSpace(override.Command) == "" { + return fmt.Errorf("acpx %s agent override command is required", agent) + } + if err := validateOverrideCommand(agent, strings.TrimSpace(override.Command), override.Args); err != nil { + return err + } + } else if _, err := os.Lstat(target); errors.Is(err, os.ErrNotExist) { + // Delete against a missing file is a no-op; skip creating the + // directory and lock file for it. + return nil + } else if err != nil { + return fmt.Errorf("inspect acpx config: %w", err) + } if err := os.MkdirAll(dir, 0o700); err != nil { return err } - payload := struct { - Agents map[string]struct { + lockPath := target + ".lock" + guard := overrideWriteLock(lockPath) + guard.Lock() + defer guard.Unlock() + lockFile, err := acquireOverrideFileLock(lockPath) + if err != nil { + return err + } + defer func() { + _ = unlockOverrideFile(lockFile) + _ = lockFile.Close() + }() + + doc := map[string]json.RawMessage{} + agents := map[string]json.RawMessage{} + info, err := os.Lstat(target) + switch { + case errors.Is(err, os.ErrNotExist): + // Fresh document; a delete below is a no-op. + case err != nil: + return fmt.Errorf("inspect acpx config: %w", err) + default: + if info.Mode()&os.ModeSymlink != 0 || !info.Mode().IsRegular() || info.Size() > maxAgentConfigBytes { + return fmt.Errorf("acpx config must be a bounded regular file") + } + data, err := os.ReadFile(target) + if err != nil { + return fmt.Errorf("read acpx config: %w", err) + } + if err := json.Unmarshal(data, &doc); err != nil { + return fmt.Errorf("parse acpx config: %w", err) + } + if doc == nil { // top-level JSON null + doc = map[string]json.RawMessage{} + } + if raw, ok := doc["agents"]; ok { + if err := json.Unmarshal(raw, &agents); err != nil { + return fmt.Errorf("parse acpx agents: %w", err) + } + if agents == nil { // "agents": null + agents = map[string]json.RawMessage{} + } + } + } + + if override == nil { + if _, present := agents[agent]; !present { + return nil + } + delete(agents, agent) + if len(agents) == 0 && len(doc) == 1 { + // The file existed solely for overrides; remove it whole rather + // than leaving an empty agents shell behind. + if err := os.Remove(target); err != nil && !errors.Is(err, os.ErrNotExist) { + return fmt.Errorf("remove acpx config: %w", err) + } + return nil + } + } else { + entry, err := json.Marshal(struct { Command string `json:"command"` Args []string `json:"args,omitempty"` - } `json:"agents"` - }{Agents: map[string]struct { - Command string `json:"command"` - Args []string `json:"args,omitempty"` - }{override.Agent: {Command: override.Command, Args: append([]string(nil), override.Args...)}}} - data, err := json.MarshalIndent(payload, "", " ") + }{Command: strings.TrimSpace(override.Command), Args: append([]string(nil), override.Args...)}) + if err != nil { + return err + } + agents[agent] = entry + } + rawAgents, err := json.Marshal(agents) + if err != nil { + return err + } + doc["agents"] = rawAgents + data, err := json.MarshalIndent(doc, "", " ") if err != nil { return err } + return writeAgentConfigAtomic(dir, target, append(data, '\n')) +} + +func overrideWriteLock(lockPath string) *sync.Mutex { + guard, _ := overrideWriteLocks.LoadOrStore(lockPath, &sync.Mutex{}) + return guard.(*sync.Mutex) +} + +func acquireOverrideFileLock(lockPath string) (*os.File, error) { + file, err := os.OpenFile(lockPath, os.O_CREATE|os.O_RDWR, 0o600) + if err != nil { + return nil, fmt.Errorf("open acpx config lock: %w", err) + } + deadline := time.Now().Add(overrideLockTimeout) + delay := 5 * time.Millisecond + for { + err := tryLockFile(file) + if err == nil { + return file, nil + } + if !lockUnavailable(err) { + _ = file.Close() + return nil, fmt.Errorf("lock acpx config: %w", err) + } + if !time.Now().Before(deadline) { + _ = file.Close() + return nil, fmt.Errorf("timed out after %s waiting for the acpx config lock", overrideLockTimeout) + } + time.Sleep(delay) + if delay < 50*time.Millisecond { + delay *= 2 + } + } +} + +// writeAgentConfigAtomic installs data at target through a temp file in the +// same directory plus rename, so a concurrent reader never observes partial +// JSON. A crash between temp creation and rename strands at most one bounded +// `.config-*` temp file per crashed apply; the next apply self-heals. +func writeAgentConfigAtomic(dir, target string, data []byte) error { temporary, err := os.CreateTemp(dir, ".config-*") if err != nil { return err @@ -113,7 +270,7 @@ func MaterializeAgentOverride(home string, override AgentOverride) error { temporary.Close() return err } - if _, err := temporary.Write(append(data, '\n')); err != nil { + if _, err := temporary.Write(data); err != nil { temporary.Close() return err } @@ -124,17 +281,20 @@ func MaterializeAgentOverride(home string, override AgentOverride) error { if err := temporary.Close(); err != nil { return err } - target := filepath.Join(dir, "config.json") - if info, err := os.Lstat(target); err == nil && (info.IsDir() || info.Mode()&os.ModeSymlink != 0) { - if err := os.RemoveAll(target); err != nil { - return err - } - } else if err != nil && !errors.Is(err, os.ErrNotExist) { - return err - } return os.Rename(name, target) } +// MaterializeAgentOverride installs one agent's override, preserving other +// agents' entries and unrelated top-level fields already present in the +// config. It is ApplyAgentOverride with an upsert; no caller can whole-file +// clobber a shared config through it. +func MaterializeAgentOverride(home string, override AgentOverride) error { + if strings.TrimSpace(home) == "" || strings.TrimSpace(override.Agent) == "" || strings.TrimSpace(override.Command) == "" { + return fmt.Errorf("acpx agent override destination and value are required") + } + return ApplyAgentOverride(home, override.Agent, &override) +} + func AgentOverrideDescription(override AgentOverride) string { full := strings.Join(append([]string{override.Command}, override.Args...), " ") if adapter := codexAdapterPattern.FindString(full); adapter != "" { diff --git a/internal/acpx/agent_override_test.go b/internal/acpx/agent_override_test.go index 464d408d..230c8fdf 100644 --- a/internal/acpx/agent_override_test.go +++ b/internal/acpx/agent_override_test.go @@ -1,11 +1,17 @@ package acpx import ( + "bytes" "encoding/json" + "errors" "os" "path/filepath" + "runtime" "strings" + "sync" + "sync/atomic" "testing" + "time" ) func TestAgentOverrideSelectivelyLoadsAndMaterializesOneCommand(t *testing.T) { @@ -59,6 +65,481 @@ func TestAgentOverrideRejectsUnboundedOrMultilineCommands(t *testing.T) { } } +// Regression for the shared-HOME race: one dispatch materializes its agent's +// host override while a peer dispatch for another agent has NO host override +// and deletes its (absent) entry. The peer's delete must never erase the +// materialized entry. At HEAD both dispatches ran on the whole file +// (remove / single-agent replace) and the override was lost. +func TestApplyAgentOverrideConcurrentUpsertAndPeerDeleteKeepsEntry(t *testing.T) { + home := t.TempDir() + qoder := AgentOverride{Agent: AgentQoder, Command: "npx", Args: []string{"-y", "@qodercode/acp@1.2.3"}} + const rounds = 50 + var wg sync.WaitGroup + wg.Add(2) + go func() { + defer wg.Done() + for i := 0; i < rounds; i++ { + if err := ApplyAgentOverride(home, AgentQoder, &qoder); err != nil { + t.Errorf("upsert qoder: %v", err) + return + } + } + }() + go func() { + defer wg.Done() + for i := 0; i < rounds; i++ { + if err := ApplyAgentOverride(home, AgentCodex, nil); err != nil { + t.Errorf("delete codex: %v", err) + return + } + } + }() + wg.Wait() + got, ok, err := LoadAgentOverride(home, AgentQoder) + if err != nil || !ok { + t.Fatalf("qoder override lost under concurrent peer delete: ok=%v err=%v", ok, err) + } + if got.Command != qoder.Command || strings.Join(got.Args, "\x00") != strings.Join(qoder.Args, "\x00") { + t.Fatalf("qoder override = %+v, want %+v", got, qoder) + } + if _, ok, err := LoadAgentOverride(home, AgentCodex); err != nil || ok { + t.Fatalf("codex entry should stay absent: ok=%v err=%v", ok, err) + } +} + +// Regression for the whole-file last-writer-wins race: two dispatches with +// different host overrides must both end up in the shared config. +func TestApplyAgentOverrideConcurrentDistinctUpsertsKeepBothEntries(t *testing.T) { + home := t.TempDir() + qoder := AgentOverride{Agent: AgentQoder, Command: "npx", Args: []string{"-y", "@qodercode/acp@1.2.3"}} + codex := AgentOverride{Agent: AgentCodex, Command: "npx", Args: []string{"-y", "@agentclientprotocol/codex-acp@1.1.2"}} + const rounds = 50 + var wg sync.WaitGroup + wg.Add(2) + go func() { + defer wg.Done() + for i := 0; i < rounds; i++ { + if err := ApplyAgentOverride(home, AgentQoder, &qoder); err != nil { + t.Errorf("upsert qoder: %v", err) + return + } + } + }() + go func() { + defer wg.Done() + for i := 0; i < rounds; i++ { + if err := ApplyAgentOverride(home, AgentCodex, &codex); err != nil { + t.Errorf("upsert codex: %v", err) + return + } + } + }() + wg.Wait() + for _, want := range []AgentOverride{qoder, codex} { + got, ok, err := LoadAgentOverride(home, want.Agent) + if err != nil || !ok { + t.Fatalf("%s override lost under concurrent upserts: ok=%v err=%v", want.Agent, ok, err) + } + if got.Command != want.Command || strings.Join(got.Args, "\x00") != strings.Join(want.Args, "\x00") { + t.Fatalf("%s override = %+v, want %+v", want.Agent, got, want) + } + } +} + +// Stress: interleaved upsert qoder / upsert codex / delete claude on one home +// while concurrent readers hammer LoadAgentOverride. Readers must never +// observe a partial document, and the final file must contain exactly +// qoder+codex. Must pass under -race. +func TestApplyAgentOverrideConcurrentStress(t *testing.T) { + home := t.TempDir() + seedConfig(t, home, `{"defaultAgent":"claude","agents":{"claude":{"command":"claude-acp","args":["--serve"]}}}`) + qoder := AgentOverride{Agent: AgentQoder, Command: "qoder-acp", Args: []string{"--stdio"}} + codex := AgentOverride{Agent: AgentCodex, Command: "codex-acp"} + const rounds = 40 + var stop atomic.Bool + var readers sync.WaitGroup + for _, agent := range []string{AgentQoder, AgentCodex, AgentClaude} { + readers.Add(1) + go func(agent string) { + defer readers.Done() + for !stop.Load() { + if _, _, err := LoadAgentOverride(home, agent); err != nil { + t.Errorf("reader(%s) observed a broken config: %v", agent, err) + return + } + } + }(agent) + } + var writers sync.WaitGroup + writers.Add(3) + go func() { + defer writers.Done() + for i := 0; i < rounds; i++ { + if err := ApplyAgentOverride(home, AgentQoder, &qoder); err != nil { + t.Errorf("upsert qoder: %v", err) + return + } + } + }() + go func() { + defer writers.Done() + for i := 0; i < rounds; i++ { + if err := ApplyAgentOverride(home, AgentCodex, &codex); err != nil { + t.Errorf("upsert codex: %v", err) + return + } + } + }() + go func() { + defer writers.Done() + for i := 0; i < rounds; i++ { + if err := ApplyAgentOverride(home, AgentClaude, nil); err != nil { + t.Errorf("delete claude: %v", err) + return + } + } + }() + writers.Wait() + stop.Store(true) + readers.Wait() + + data, err := os.ReadFile(configPath(home)) + if err != nil { + t.Fatal(err) + } + var doc struct { + Agents map[string]json.RawMessage `json:"agents"` + } + if err := json.Unmarshal(data, &doc); err != nil { + t.Fatalf("final config is not valid JSON: %v", err) + } + if len(doc.Agents) != 2 { + t.Fatalf("final agents = %v, want exactly qoder+codex", agentNames(doc.Agents)) + } + for _, agent := range []string{AgentQoder, AgentCodex} { + if _, ok := doc.Agents[agent]; !ok { + t.Fatalf("final config missing %s entry: %s", agent, data) + } + } + info, err := os.Stat(configPath(home)) + if err != nil { + t.Fatal(err) + } + if info.Mode().Perm() != 0o600 { + t.Fatalf("final config mode = %v, want 0600", info.Mode().Perm()) + } +} + +// Unrelated top-level fields and other agents' raw entries must survive an +// upsert byte-for-byte (modulo JSON-insignificant whitespace). +func TestApplyAgentOverridePreservesUnrelatedFieldsAndPeerAgents(t *testing.T) { + home := t.TempDir() + other := `{"command":"keep-me","args":["--x"],"extra":{"nested":[1,2,{"a":"b"}]}}` + seedConfig(t, home, `{"defaultAgent":"claude","ttl":30,"custom":{"nested":[1,2,{"a":"b"}]},"agents":{"other":`+other+`}}`) + qoder := AgentOverride{Agent: AgentQoder, Command: "qoder-acp", Args: []string{"--stdio"}} + if err := ApplyAgentOverride(home, AgentQoder, &qoder); err != nil { + t.Fatal(err) + } + data, err := os.ReadFile(configPath(home)) + if err != nil { + t.Fatal(err) + } + var doc map[string]json.RawMessage + if err := json.Unmarshal(data, &doc); err != nil { + t.Fatal(err) + } + assertRawJSONEqual(t, doc["defaultAgent"], `"claude"`) + assertRawJSONEqual(t, doc["ttl"], `30`) + assertRawJSONEqual(t, doc["custom"], `{"nested":[1,2,{"a":"b"}]}`) + var agents map[string]json.RawMessage + if err := json.Unmarshal(doc["agents"], &agents); err != nil { + t.Fatal(err) + } + assertRawJSONEqual(t, agents["other"], other) + assertRawJSONEqual(t, agents[AgentQoder], `{"command":"qoder-acp","args":["--stdio"]}`) +} + +// A delete that empties agents removes the file only when it existed solely +// for overrides; any other top-level key keeps the file (minus the entry). +func TestApplyAgentOverrideDeleteRemovesAgentsOnlyFile(t *testing.T) { + home := t.TempDir() + seedConfig(t, home, `{"agents":{"claude":{"command":"claude-acp"}}}`) + if err := ApplyAgentOverride(home, AgentClaude, nil); err != nil { + t.Fatal(err) + } + if _, err := os.Lstat(configPath(home)); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("agents-only config should have been removed: %v", err) + } +} + +func TestApplyAgentOverrideDeleteKeepsFileWithOtherContent(t *testing.T) { + home := t.TempDir() + seedConfig(t, home, `{"defaultAgent":"qoder","agents":{"claude":{"command":"claude-acp"},"qoder":{"command":"qoder-acp"}}}`) + if err := ApplyAgentOverride(home, AgentClaude, nil); err != nil { + t.Fatal(err) + } + data, err := os.ReadFile(configPath(home)) + if err != nil { + t.Fatal(err) + } + var doc map[string]json.RawMessage + if err := json.Unmarshal(data, &doc); err != nil { + t.Fatal(err) + } + assertRawJSONEqual(t, doc["defaultAgent"], `"qoder"`) + var agents map[string]json.RawMessage + if err := json.Unmarshal(doc["agents"], &agents); err != nil { + t.Fatal(err) + } + if _, ok := agents[AgentClaude]; ok { + t.Fatalf("claude entry survived delete: %s", data) + } + assertRawJSONEqual(t, agents[AgentQoder], `{"command":"qoder-acp"}`) +} + +// Delete of an agent that is not present is a no-op: the file is not +// rewritten at all. +func TestApplyAgentOverrideDeleteAbsentEntryIsNoOp(t *testing.T) { + home := t.TempDir() + seed := `{"agents":{"qoder":{"command":"qoder-acp"}}}` + seedConfig(t, home, seed) + if err := ApplyAgentOverride(home, AgentCodex, nil); err != nil { + t.Fatal(err) + } + data, err := os.ReadFile(configPath(home)) + if err != nil { + t.Fatal(err) + } + if string(data) != seed { + t.Fatalf("no-op delete rewrote the config: %q -> %q", seed, data) + } +} + +func TestApplyAgentOverrideMissingFile(t *testing.T) { + home := t.TempDir() + if err := ApplyAgentOverride(home, AgentCodex, nil); err != nil { + t.Fatalf("delete on missing file = %v", err) + } + if _, err := os.Lstat(filepath.Join(home, ".acpx")); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("no-op delete should not create .acpx: %v", err) + } + qoder := AgentOverride{Agent: AgentQoder, Command: "qoder-acp"} + if err := ApplyAgentOverride(home, AgentQoder, &qoder); err != nil { + t.Fatal(err) + } + dirInfo, err := os.Stat(filepath.Join(home, ".acpx")) + if err != nil { + t.Fatal(err) + } + if dirInfo.Mode().Perm() != 0o700 { + t.Fatalf(".acpx dir mode = %v, want 0700", dirInfo.Mode().Perm()) + } + fileInfo, err := os.Stat(configPath(home)) + if err != nil { + t.Fatal(err) + } + if fileInfo.Mode().Perm() != 0o600 { + t.Fatalf("config mode = %v, want 0600", fileInfo.Mode().Perm()) + } + if _, ok, err := LoadAgentOverride(home, AgentQoder); err != nil || !ok { + t.Fatalf("upserted override not loadable: ok=%v err=%v", ok, err) + } +} + +// Fail closed: a malformed, oversized, symlinked, or directory config must +// produce an error and be left byte-for-byte untouched. +func TestApplyAgentOverrideMalformedConfigFailsClosed(t *testing.T) { + home := t.TempDir() + body := `{"agents": broken` + seedConfig(t, home, body) + qoder := AgentOverride{Agent: AgentQoder, Command: "qoder-acp"} + if err := ApplyAgentOverride(home, AgentQoder, &qoder); err == nil { + t.Fatal("malformed config accepted") + } + assertConfigBytes(t, home, body) + assertNoStaleTempFiles(t, home) +} + +func TestApplyAgentOverrideOversizedConfigFailsClosed(t *testing.T) { + home := t.TempDir() + body := `{"agents":{}}` + strings.Repeat(" ", maxAgentConfigBytes) + seedConfig(t, home, body) + qoder := AgentOverride{Agent: AgentQoder, Command: "qoder-acp"} + if err := ApplyAgentOverride(home, AgentQoder, &qoder); err == nil { + t.Fatal("oversized config accepted") + } + assertConfigBytes(t, home, body) +} + +func TestApplyAgentOverrideSymlinkConfigFailsClosed(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("symlink creation requires privileges on windows") + } + home := t.TempDir() + if err := os.MkdirAll(filepath.Join(home, ".acpx"), 0o700); err != nil { + t.Fatal(err) + } + real := filepath.Join(home, "elsewhere.json") + if err := os.WriteFile(real, []byte(`{"agents":{}}`), 0o600); err != nil { + t.Fatal(err) + } + if err := os.Symlink(real, configPath(home)); err != nil { + t.Fatal(err) + } + qoder := AgentOverride{Agent: AgentQoder, Command: "qoder-acp"} + if err := ApplyAgentOverride(home, AgentQoder, &qoder); err == nil { + t.Fatal("symlinked config accepted") + } + info, err := os.Lstat(configPath(home)) + if err != nil { + t.Fatalf("symlink was replaced: %v", err) + } + if info.Mode()&os.ModeSymlink == 0 { + t.Fatalf("symlink was replaced: mode=%v", info.Mode()) + } + if data, err := os.ReadFile(real); err != nil || string(data) != `{"agents":{}}` { + t.Fatalf("symlink target modified: %q err=%v", data, err) + } +} + +func TestApplyAgentOverrideDirectoryConfigFailsClosed(t *testing.T) { + home := t.TempDir() + if err := os.MkdirAll(configPath(home), 0o700); err != nil { + t.Fatal(err) + } + qoder := AgentOverride{Agent: AgentQoder, Command: "qoder-acp"} + if err := ApplyAgentOverride(home, AgentQoder, &qoder); err == nil { + t.Fatal("directory config accepted") + } + if info, err := os.Lstat(configPath(home)); err != nil || !info.IsDir() { + t.Fatalf("config directory disturbed: err=%v", err) + } +} + +// After ApplyAgentOverride returns, the lock file must be unlocked and no +// temp files may be stranded in .acpx. +func TestApplyAgentOverrideReleasesLockAndLeavesNoTempFiles(t *testing.T) { + home := t.TempDir() + qoder := AgentOverride{Agent: AgentQoder, Command: "qoder-acp"} + if err := ApplyAgentOverride(home, AgentQoder, &qoder); err != nil { + t.Fatal(err) + } + if err := ApplyAgentOverride(home, AgentQoder, nil); err != nil { + t.Fatal(err) + } + lockPath := configPath(home) + ".lock" + info, err := os.Stat(lockPath) + if err != nil { + t.Fatal(err) + } + if info.Mode().Perm() != 0o600 { + t.Fatalf("lock file mode = %v, want 0600", info.Mode().Perm()) + } + file, err := os.OpenFile(lockPath, os.O_RDWR, 0) + if err != nil { + t.Fatal(err) + } + defer file.Close() + if err := tryLockFile(file); err != nil { + t.Fatalf("config lock still held after ApplyAgentOverride returned: %v", err) + } + if err := unlockOverrideFile(file); err != nil { + t.Fatal(err) + } + assertNoStaleTempFiles(t, home) +} + +// A lock held by another handle must produce a bounded wait and a clear +// error, never an indefinite block. +func TestApplyAgentOverrideContendedLockTimesOut(t *testing.T) { + home := t.TempDir() + qoder := AgentOverride{Agent: AgentQoder, Command: "qoder-acp"} + if err := ApplyAgentOverride(home, AgentQoder, &qoder); err != nil { + t.Fatal(err) + } + lockPath := configPath(home) + ".lock" + holder, err := os.OpenFile(lockPath, os.O_RDWR, 0) + if err != nil { + t.Fatal(err) + } + defer holder.Close() + if err := tryLockFile(holder); err != nil { + t.Fatalf("acquire lock for contention setup: %v", err) + } + defer unlockOverrideFile(holder) + start := time.Now() + err = ApplyAgentOverride(home, AgentQoder, &qoder) + elapsed := time.Since(start) + if err == nil || !strings.Contains(err.Error(), "timed out") { + t.Fatalf("contended apply error = %v, want a lock timeout", err) + } + if elapsed > overrideLockTimeout+5*time.Second { + t.Fatalf("contended apply blocked for %s, want a bounded wait", elapsed) + } +} + +func configPath(home string) string { + return filepath.Join(home, ".acpx", "config.json") +} + +func seedConfig(t *testing.T, home, body string) { + t.Helper() + if err := os.MkdirAll(filepath.Join(home, ".acpx"), 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(configPath(home), []byte(body), 0o600); err != nil { + t.Fatal(err) + } +} + +func assertConfigBytes(t *testing.T, home, want string) { + t.Helper() + data, err := os.ReadFile(configPath(home)) + if err != nil { + t.Fatal(err) + } + if string(data) != want { + t.Fatalf("config modified on failure path:\n got: %q\nwant: %q", data, want) + } +} + +func assertNoStaleTempFiles(t *testing.T, home string) { + t.Helper() + entries, err := os.ReadDir(filepath.Join(home, ".acpx")) + if err != nil { + t.Fatal(err) + } + for _, entry := range entries { + if strings.HasPrefix(entry.Name(), ".config-") { + t.Fatalf("stale temp file left behind: %s", entry.Name()) + } + } +} + +func assertRawJSONEqual(t *testing.T, got json.RawMessage, want string) { + t.Helper() + if !bytes.Equal(compactJSON(t, got), compactJSON(t, json.RawMessage(want))) { + t.Fatalf("raw JSON mismatch:\n got: %s\nwant: %s", got, want) + } +} + +func compactJSON(t *testing.T, raw json.RawMessage) []byte { + t.Helper() + var out bytes.Buffer + if err := json.Compact(&out, raw); err != nil { + t.Fatalf("invalid JSON %q: %v", raw, err) + } + return out.Bytes() +} + +func agentNames(agents map[string]json.RawMessage) []string { + names := make([]string, 0, len(agents)) + for name := range agents { + names = append(names, name) + } + return names +} + func mustJSON(t *testing.T, value string) []byte { t.Helper() data, err := json.Marshal(value) diff --git a/internal/acpx/filelock_unix.go b/internal/acpx/filelock_unix.go new file mode 100644 index 00000000..92d1e7dc --- /dev/null +++ b/internal/acpx/filelock_unix.go @@ -0,0 +1,21 @@ +//go:build !windows + +package acpx + +import ( + "errors" + "os" + "syscall" +) + +func tryLockFile(file *os.File) error { + return syscall.Flock(int(file.Fd()), syscall.LOCK_EX|syscall.LOCK_NB) +} + +func unlockOverrideFile(file *os.File) error { + return syscall.Flock(int(file.Fd()), syscall.LOCK_UN) +} + +func lockUnavailable(err error) bool { + return errors.Is(err, syscall.EWOULDBLOCK) || errors.Is(err, syscall.EAGAIN) +} diff --git a/internal/acpx/filelock_windows.go b/internal/acpx/filelock_windows.go new file mode 100644 index 00000000..e7dddca3 --- /dev/null +++ b/internal/acpx/filelock_windows.go @@ -0,0 +1,26 @@ +//go:build windows + +package acpx + +import ( + "errors" + "os" + + "golang.org/x/sys/windows" +) + +const overrideLockByteCount = 1 + +func tryLockFile(file *os.File) error { + var overlapped windows.Overlapped + return windows.LockFileEx(windows.Handle(file.Fd()), windows.LOCKFILE_EXCLUSIVE_LOCK|windows.LOCKFILE_FAIL_IMMEDIATELY, 0, overrideLockByteCount, 0, &overlapped) +} + +func unlockOverrideFile(file *os.File) error { + var overlapped windows.Overlapped + return windows.UnlockFileEx(windows.Handle(file.Fd()), 0, overrideLockByteCount, 0, &overlapped) +} + +func lockUnavailable(err error) bool { + return errors.Is(err, windows.ERROR_LOCK_VIOLATION) +} diff --git a/internal/commentrunner/jobs/dispatcher.go b/internal/commentrunner/jobs/dispatcher.go index bcf06e4b..56f9caee 100644 --- a/internal/commentrunner/jobs/dispatcher.go +++ b/internal/commentrunner/jobs/dispatcher.go @@ -1990,6 +1990,12 @@ func rejectRepositoryAcpxConfig(workspacePath string) error { return nil } +// materializeHostAcpxAgentOverride refreshes exactly one agent's entry in the +// shared runtime HOME. The host config stays authoritative for this agent, but +// the write must be a per-agent merge applied under lock: concurrent +// dispatches in the same runner scope share cfg.TempHome, so a whole-file +// remove or replace would erase a peer agent's concurrently materialized +// override (an absent host override deletes only this agent's entry). func materializeHostAcpxAgentOverride(cfg *sandbox.Config, agent string) error { if cfg == nil || strings.TrimSpace(agent) == "" || strings.TrimSpace(cfg.TempHome) == "" { return nil @@ -1999,14 +2005,13 @@ func materializeHostAcpxAgentOverride(cfg *sandbox.Config, agent string) error { if err != nil { return fmt.Errorf("load host acpx %s agent override: %w", agent, err) } - target := filepath.Join(cfg.TempHome, ".acpx", "config.json") if !ok { - if err := os.Remove(target); err != nil && !errors.Is(err, os.ErrNotExist) { - return fmt.Errorf("remove stale runtime acpx agent override: %w", err) + if err := acpx.ApplyAgentOverride(cfg.TempHome, agent, nil); err != nil { + return fmt.Errorf("remove stale runtime acpx %s agent override: %w", agent, err) } return nil } - if err := acpx.MaterializeAgentOverride(cfg.TempHome, override); err != nil { + if err := acpx.ApplyAgentOverride(cfg.TempHome, agent, &override); err != nil { return fmt.Errorf("materialize host acpx %s agent override: %w", agent, err) } return nil diff --git a/internal/commentrunner/jobs/dispatcher_test.go b/internal/commentrunner/jobs/dispatcher_test.go index 215e7e10..26c05f98 100644 --- a/internal/commentrunner/jobs/dispatcher_test.go +++ b/internal/commentrunner/jobs/dispatcher_test.go @@ -1859,6 +1859,87 @@ func TestHostSSHRuntimeLoadsAcpxOverrideFromReusedHome(t *testing.T) { } } +// Concurrent dispatches in one runner scope share the runtime HOME, so +// per-agent override refreshes must merge: two agents with distinct host +// overrides materialized at the same time must both end up in the shared +// config (whole-file last-writer-wins would silently drop one). +func TestMaterializeHostAcpxAgentOverrideConcurrentDistinctAgents(t *testing.T) { + hostHome := t.TempDir() + if err := os.MkdirAll(filepath.Join(hostHome, ".acpx"), 0o700); err != nil { + t.Fatal(err) + } + hostConfig := `{"agents":{"qoder":{"command":"npx","args":["-y","@qodercode/acp@1.2.3"]},"codex":{"command":"npx","args":["-y","@agentclientprotocol/codex-acp@1.1.2"]}}}` + if err := os.WriteFile(filepath.Join(hostHome, ".acpx", "config.json"), []byte(hostConfig), 0o600); err != nil { + t.Fatal(err) + } + runtimeHome := t.TempDir() + cfg := sandbox.Config{HostEnv: []string{"HOME=" + hostHome}, TempHome: runtimeHome} + const rounds = 40 + var wg sync.WaitGroup + for _, agent := range []string{acpx.AgentQoder, acpx.AgentCodex} { + wg.Add(1) + go func(agent string) { + defer wg.Done() + for i := 0; i < rounds; i++ { + if err := materializeHostAcpxAgentOverride(&cfg, agent); err != nil { + t.Errorf("materialize %s override: %v", agent, err) + return + } + } + }(agent) + } + wg.Wait() + for _, agent := range []string{acpx.AgentQoder, acpx.AgentCodex} { + if _, ok, err := acpx.LoadAgentOverride(runtimeHome, agent); err != nil || !ok { + t.Fatalf("%s override lost in shared runtime HOME: ok=%v err=%v", agent, ok, err) + } + } +} + +// The peer-delete race: one dispatch materializes its host override while a +// concurrent dispatch for another agent has NO host override and only prunes +// its own (absent) entry. The materialized entry must survive. +func TestMaterializeHostAcpxAgentOverrideConcurrentUpsertVsNoOverride(t *testing.T) { + hostHome := t.TempDir() + if err := os.MkdirAll(filepath.Join(hostHome, ".acpx"), 0o700); err != nil { + t.Fatal(err) + } + hostConfig := `{"agents":{"qoder":{"command":"npx","args":["-y","@qodercode/acp@1.2.3"]}}}` + if err := os.WriteFile(filepath.Join(hostHome, ".acpx", "config.json"), []byte(hostConfig), 0o600); err != nil { + t.Fatal(err) + } + runtimeHome := t.TempDir() + cfg := sandbox.Config{HostEnv: []string{"HOME=" + hostHome}, TempHome: runtimeHome} + const rounds = 40 + var wg sync.WaitGroup + wg.Add(2) + go func() { + defer wg.Done() + for i := 0; i < rounds; i++ { + if err := materializeHostAcpxAgentOverride(&cfg, acpx.AgentQoder); err != nil { + t.Errorf("materialize qoder override: %v", err) + return + } + } + }() + go func() { + defer wg.Done() + for i := 0; i < rounds; i++ { + if err := materializeHostAcpxAgentOverride(&cfg, acpx.AgentCodex); err != nil { + t.Errorf("prune absent codex override: %v", err) + return + } + } + }() + wg.Wait() + if _, ok, err := acpx.LoadAgentOverride(runtimeHome, acpx.AgentQoder); err != nil || !ok { + t.Fatalf("qoder override erased by peer no-override dispatch: ok=%v err=%v", ok, err) + } + if _, ok, err := acpx.LoadAgentOverride(runtimeHome, acpx.AgentCodex); err != nil || ok { + t.Fatalf("codex entry should stay absent: ok=%v err=%v", ok, err) + } +} + func TestSandboxRunnerRejectsRepositoryAcpxConfig(t *testing.T) { root := t.TempDir() workspacePath := filepath.Join(root, "workspace") From fb3ae0329150648aab7f1b29f24007c717e42b23 Mon Sep 17 00:00:00 2001 From: johnlanni Date: Sun, 9 Aug 2026 11:46:02 +0800 Subject: [PATCH 17/17] test(runner): deterministic acpx override peer-remove regression Add a sequential regression test that drives the production materializeHostAcpxAgentOverride path: materialize a qoder override into the shared runtime HOME, then prune codex with no host override, and assert the qoder entry still parses from the shared HOME. The pre-fix whole-file remove deleted the shared config.json outright, so this fails deterministically there, unlike the concurrent cover. Also note at the keyed-mutex sync.Map that entries persist for the process lifetime (one per distinct lock path; bounded by stable runner homes, with per-call accrual only on the ad-hoc MkdirTemp fallback). Signed-off-by: johnlanni --- internal/acpx/agent_override.go | 6 +++- .../commentrunner/jobs/dispatcher_test.go | 35 +++++++++++++++++++ 2 files changed, 40 insertions(+), 1 deletion(-) diff --git a/internal/acpx/agent_override.go b/internal/acpx/agent_override.go index e6f85b75..01321e65 100644 --- a/internal/acpx/agent_override.go +++ b/internal/acpx/agent_override.go @@ -26,7 +26,11 @@ var codexAdapterPattern = regexp.MustCompile(`@agentclientprotocol/codex-acp@[A- // overrideWriteLocks serializes in-process writers per config file. The flock // on config.json.lock additionally serializes cooperating processes (e.g. a -// runner and an operator CLI sharing one HOME). +// runner and an operator CLI sharing one HOME). Entries persist for the +// process lifetime: one per distinct lock path, bounded in practice by the +// dispatcher's stable runner homes. Ad-hoc callers on the MkdirTemp fallback +// path can accrue one entry per call; accepted, since growth stays +// process-lifetime bounded. var overrideWriteLocks sync.Map // map[string]*sync.Mutex, keyed by cleaned lock path type AgentOverride struct { diff --git a/internal/commentrunner/jobs/dispatcher_test.go b/internal/commentrunner/jobs/dispatcher_test.go index 26c05f98..be23e054 100644 --- a/internal/commentrunner/jobs/dispatcher_test.go +++ b/internal/commentrunner/jobs/dispatcher_test.go @@ -1940,6 +1940,41 @@ func TestMaterializeHostAcpxAgentOverrideConcurrentUpsertVsNoOverride(t *testing } } +// Deterministic sequential cover for the peer-remove scenario raced above: +// one dispatch materializes its host qoder override into the shared runtime +// HOME, then a dispatch for codex with NO host override prunes its own +// (absent) entry. The pre-fix whole-file remove deleted the shared +// config.json outright, erasing the peer qoder entry, so this fails there; +// the per-agent delete must leave the qoder entry parseable. +func TestMaterializeHostAcpxAgentOverridePeerRemoveKeepsOtherAgent(t *testing.T) { + hostHome := t.TempDir() + if err := os.MkdirAll(filepath.Join(hostHome, ".acpx"), 0o700); err != nil { + t.Fatal(err) + } + hostConfig := `{"agents":{"qoder":{"command":"npx","args":["-y","@qodercode/acp@1.2.3"]}}}` + if err := os.WriteFile(filepath.Join(hostHome, ".acpx", "config.json"), []byte(hostConfig), 0o600); err != nil { + t.Fatal(err) + } + runtimeHome := t.TempDir() + cfg := sandbox.Config{HostEnv: []string{"HOME=" + hostHome}, TempHome: runtimeHome} + if err := materializeHostAcpxAgentOverride(&cfg, acpx.AgentQoder); err != nil { + t.Fatalf("materialize qoder override: %v", err) + } + if err := materializeHostAcpxAgentOverride(&cfg, acpx.AgentCodex); err != nil { + t.Fatalf("prune absent codex override: %v", err) + } + override, ok, err := acpx.LoadAgentOverride(runtimeHome, acpx.AgentQoder) + if err != nil || !ok { + t.Fatalf("qoder override erased by peer no-override dispatch: ok=%v err=%v", ok, err) + } + if override.Command != "npx" || len(override.Args) != 2 || override.Args[1] != "@qodercode/acp@1.2.3" { + t.Fatalf("qoder override = %+v", override) + } + if _, ok, err := acpx.LoadAgentOverride(runtimeHome, acpx.AgentCodex); err != nil || ok { + t.Fatalf("codex entry should stay absent: ok=%v err=%v", ok, err) + } +} + func TestSandboxRunnerRejectsRepositoryAcpxConfig(t *testing.T) { root := t.TempDir() workspacePath := filepath.Join(root, "workspace")